diff --git a/alpine/configure-system.sh b/alpine/configure-system.sh index b6edf64..d84279c 100755 --- a/alpine/configure-system.sh +++ b/alpine/configure-system.sh @@ -7,6 +7,7 @@ rc-update add devfs sysinit rc-update add procfs sysinit rc-update add sysfs sysinit rc-update add cgroups sysinit +rc-update add hwclock boot rc-update add fancontrol boot rc-update add loopback boot rc-update add hostname boot diff --git a/alpine/install-packages.sh b/alpine/install-packages.sh index 9b26dc6..21469e8 100755 --- a/alpine/install-packages.sh +++ b/alpine/install-packages.sh @@ -31,7 +31,8 @@ if [ $? -ne 0 ]; then exit $? fi -sed -i "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" /etc/crio/crio.conf.d/10-crio.conf +sed -i -e "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" \ + /etc/crio/crio.conf.d/10-crio.conf grep cgroup_manager /etc/crio/crio.conf.d/10-crio.conf || exit 1 echo "##################################################### Installing Control Agent" diff --git a/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf b/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf new file mode 100644 index 0000000..52a3811 --- /dev/null +++ b/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf @@ -0,0 +1,4 @@ +[crio.runtime] +default_sysctls = [ + "net.ipv4.ip_unprivileged_port_start=20" +] diff --git a/build.env b/build.env index 979202f..872115c 100644 --- a/build.env +++ b/build.env @@ -5,8 +5,8 @@ TAG=dev # The Linux kernel, from NXP NXP_VERSION=lf-6.18.2-1.0.0 -CRIO_VERSION=cri-o.arm64.v1.35.1 -KUBE_VERSION=v1.35.1 +CRIO_VERSION=cri-o.arm64.v1.35.2 +KUBE_VERSION=v1.35.3 # Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one # We need fsl-ls1046a-rdb-sdk.dtb here @@ -21,6 +21,7 @@ CROSS_COMPILE=aarch64-linux-gnu- # Tools for initramfs BUSYBOX_VERSION=1_36_1 E2FSPROGS_VERSION=1.47.4 +UBOOT_VERSION=v2026.01 ## Alpine Linux ALPINE_VER=3.23.3 diff --git a/clitools/Dockerfile b/clitools/docker/ctl-agent.Dockerfile similarity index 83% rename from clitools/Dockerfile rename to clitools/docker/ctl-agent.Dockerfile index d6d9765..7387bf2 100644 --- a/clitools/Dockerfile +++ b/clitools/docker/ctl-agent.Dockerfile @@ -7,6 +7,8 @@ ENV VERSION=${VERSION} WORKDIR / COPY bin/ctl-linux-aarch64-${VERSION} ./ctl +COPY out/fw_printenv ./ +COPY out/fw_setenv ./ COPY --from=cacerts /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ ENTRYPOINT ["/ctl"] diff --git a/clitools/docker/uboot-tools.Dockerfile b/clitools/docker/uboot-tools.Dockerfile new file mode 100644 index 0000000..67c5b64 --- /dev/null +++ b/clitools/docker/uboot-tools.Dockerfile @@ -0,0 +1,41 @@ +FROM alpine:3.22 AS build + +RUN apk add --no-cache \ + build-base \ + bison \ + flex \ + linux-headers \ + file \ + binutils \ + tar + +WORKDIR /src + +ARG UBOOT_TAR +ARG UBOOT_VERSION + +COPY ${UBOOT_TAR} /tmp/ +RUN tar zxf "/tmp/$(basename "${UBOOT_TAR}")" + +RUN make tools-only_defconfig + +# Build the env tools using the supported target. +RUN make -j"$(nproc)" \ + HOSTCC=gcc \ + HOSTLD=gcc \ + HOSTCFLAGS='-O2' \ + HOSTLDFLAGS='-static' \ + envtools + +# fw_setenv is the same program; create the link ourselves. +RUN ln -sf fw_printenv tools/env/fw_setenv + +RUN file tools/env/fw_printenv tools/env/fw_setenv + +RUN readelf -d tools/env/fw_printenv || true + +RUN ! readelf -d tools/env/fw_printenv | grep -q '(NEEDED)' + +FROM scratch AS export +COPY --from=build /src/tools/env/fw_printenv /fw_printenv +COPY --from=build /src/tools/env/fw_setenv /fw_setenv diff --git a/clitools/makefile b/clitools/makefile index d55fce5..d353f3b 100644 --- a/clitools/makefile +++ b/clitools/makefile @@ -1,17 +1,24 @@ # Should be the same as upstream version in production VERSION ?= dev +UBOOT_VERSION=v2026.01 # Target kube version KUBE_VERSION ?= v1.35.0 GIT_REV := $(shell git rev-parse HEAD) -BIN_DIR := bin -OUT_DIR := out +PACKAGES_DIR := packages +BIN_DIR := bin +OUT_DIR := out + +UBOOT_TAR := $(PACKAGES_DIR)/uboot-$(UBOOT_VERSION).tar.gz BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go CRD_PATHS := ./pkg/apis/... +$(PACKAGES_DIR): + mkdir -p $@ + # Never cache this .buildinfo: @mkdir -p $(dir $(BUILDINFO_FILE)) @@ -26,13 +33,27 @@ CRD_PATHS := ./pkg/apis/... ')' \ > $(BUILDINFO_FILE) +$(UBOOT_TAR): | $(PACKAGES_DIR) + git clone --depth 1 --branch v2026.01 --filter=blob:none https://github.com/u-boot/u-boot.git $(OUT_DIR)/u-boot-$(UBOOT_VERSION) + tar -C "$(OUT_DIR)/u-boot-$(UBOOT_VERSION)" -zcf "$@" . + rm -rf $(OUT_DIR)/u-boot-$(UBOOT_VERSION) + test -f $@ + +uboot-tools: $(UBOOT_TAR) + docker buildx build --platform linux/arm64 \ + -f docker/uboot-tools.Dockerfile \ + --build-arg UBOOT_VERSION=$(UBOOT_VERSION) \ + --build-arg UBOOT_TAR=$(UBOOT_TAR) \ + --output type=local,dest=./$(OUT_DIR) . + build: .buildinfo mkdir -p $(BIN_DIR) $(OUT_DIR)/crds controller-gen crd paths=$(CRD_PATHS) output:crd:dir=$(OUT_DIR)/crds GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/ctl-linux-aarch64-$(VERSION) ./cmd/ctl -build-agent: build +build-agent: build uboot-tools docker build \ + -f docker/ctl-agent.Dockerfile \ --platform=linux/arm64 \ --build-arg VERSION=$(VERSION) \ -t localhost/monok8s/control-agent:$(VERSION) . @@ -52,4 +73,4 @@ clean: all: build build-agent build-local -.PHONY: clean all run .buildinfo build build-local build-agent +.PHONY: clean all run .buildinfo build build-local build-agent uboot-tools diff --git a/clitools/pkg/bootstrap/runner.go b/clitools/pkg/bootstrap/runner.go index 8ccb172..e57d461 100644 --- a/clitools/pkg/bootstrap/runner.go +++ b/clitools/pkg/bootstrap/runner.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - "k8s.io/klog/v2" monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" "example.com/monok8s/pkg/node" "example.com/monok8s/pkg/system" + "k8s.io/klog/v2" ) type Runner struct { @@ -132,6 +132,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner { Name: "Apply node metadata", Desc: "Apply labels/annotations to the local node if API server is reachable", }, + { + RegKey: "ConfigureUBootCommands", + Name: "Ensure fw_env config and u-boot-tools availablilty", + Desc: "Install or generate /etc/fw_env.config for U-Boot environment access", + }, { RegKey: "ApplyControlAgentDaemonSetResources", Name: "Apply daemonset for control agent", diff --git a/clitools/pkg/cmd/internal/fwprintenv.go b/clitools/pkg/cmd/internal/fwprintenv.go new file mode 100644 index 0000000..9bc6fde --- /dev/null +++ b/clitools/pkg/cmd/internal/fwprintenv.go @@ -0,0 +1,89 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + "example.com/monok8s/pkg/system" +) + +func newInternalFWPrintEnvCmd() *cobra.Command { + var key string + var configPath string + + cmd := &cobra.Command{ + Use: "fw-printenv", + Short: "Run fw_printenv", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + key = strings.TrimSpace(key) + configPath = strings.TrimSpace(configPath) + + if configPath == "" { + configPath = defaultFWEnvConfigPath + } + + if _, err := os.Stat(configPath); err != nil { + return fmt.Errorf("stat fw env config %q: %w", configPath, err) + } + + runner := system.NewRunner(system.RunnerConfig{ + DefaultTimeout: 15 * time.Second, + StreamOutput: false, + Logger: &system.StdLogger{}, + }) + + runArgs := []string{"-c", configPath} + if key != "" { + runArgs = append(runArgs, key) + } + + res, err := runner.RunWithOptions( + ctx, + "/fw_printenv", + runArgs, + system.RunOptions{ + Quiet: true, + }, + ) + if err != nil { + if res != nil { + klog.ErrorS(err, "fw_printenv failed", + "key", key, + "stdout", strings.TrimSpace(res.Stdout), + "stderr", strings.TrimSpace(res.Stderr), + ) + } + return err + } + + stdout := strings.TrimSpace(res.Stdout) + stderr := strings.TrimSpace(res.Stderr) + + if stdout != "" { + fmt.Println(stdout) + } + if stderr != "" { + klog.InfoS("fw_printenv stderr", "output", stderr) + } + + klog.InfoS("fw_printenv succeeded", + "key", key, + "configPath", configPath, + ) + + return nil + }, + } + + cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name to print") + cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config") + + return cmd +} diff --git a/clitools/pkg/cmd/internal/fwsetenv.go b/clitools/pkg/cmd/internal/fwsetenv.go new file mode 100644 index 0000000..3537e33 --- /dev/null +++ b/clitools/pkg/cmd/internal/fwsetenv.go @@ -0,0 +1,115 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + "example.com/monok8s/pkg/system" +) + +const defaultFWEnvConfigPath = "/host/etc/fw_env.config" + +func newInternalFWSetEnvCmd() *cobra.Command { + var key string + var value string + var configPath string + + cmd := &cobra.Command{ + Use: "fw-setenv", + Short: "Run fw_setenv", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + configPath = strings.TrimSpace(configPath) + + if key == "" { + return fmt.Errorf("--key is required") + } + if value == "" { + return fmt.Errorf("--value is required") + } + if configPath == "" { + configPath = defaultFWEnvConfigPath + } + + if _, err := os.Stat(configPath); err != nil { + return fmt.Errorf("stat fw env config %q: %w", configPath, err) + } + + runner := system.NewRunner(system.RunnerConfig{ + DefaultTimeout: 15 * time.Second, + StreamOutput: false, + Logger: &system.StdLogger{}, + }) + + // Preflight first so failure is clearer than blindly writing. + preflightRes, err := runner.RunWithOptions( + ctx, + "/fw_printenv", + []string{"-c", configPath}, + system.RunOptions{ + Quiet: true, + }, + ) + if err != nil { + if preflightRes != nil { + klog.ErrorS(err, "fw_printenv preflight failed", + "stdout", strings.TrimSpace(preflightRes.Stdout), + "stderr", strings.TrimSpace(preflightRes.Stderr), + ) + } + return fmt.Errorf("fw_printenv preflight: %w", err) + } + + res, err := runner.RunWithOptions( + ctx, + "/fw_setenv", + []string{ + "-c", configPath, + key, value, + }, + system.RunOptions{ + Quiet: true, + }, + ) + if err != nil { + if res != nil { + klog.ErrorS(err, "fw_setenv failed", + "key", key, + "value", value, + "stdout", strings.TrimSpace(res.Stdout), + "stderr", strings.TrimSpace(res.Stderr), + ) + } + return err + } + + if strings.TrimSpace(res.Stdout) != "" { + klog.InfoS("fw_setenv stdout", "output", strings.TrimSpace(res.Stdout)) + } + if strings.TrimSpace(res.Stderr) != "" { + klog.InfoS("fw_setenv stderr", "output", strings.TrimSpace(res.Stderr)) + } + + klog.InfoS("fw_setenv succeeded", + "key", key, + "value", value, + "configPath", configPath, + ) + return nil + }, + } + + cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name") + cmd.Flags().StringVar(&value, "value", "", "U-Boot environment variable value") + cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config") + + return cmd +} diff --git a/clitools/pkg/cmd/internal/internal.go b/clitools/pkg/cmd/internal/internal.go index 7394077..de8410c 100644 --- a/clitools/pkg/cmd/internal/internal.go +++ b/clitools/pkg/cmd/internal/internal.go @@ -1,9 +1,9 @@ package internal import ( - "github.com/spf13/cobra" "example.com/monok8s/pkg/bootstrap" "example.com/monok8s/pkg/config" + "github.com/spf13/cobra" ) func NewCmdInternal() *cobra.Command { @@ -33,6 +33,9 @@ func NewCmdInternal() *cobra.Command { return nil }, }) + cmd.AddCommand(newInternalFWSetEnvCmd()) + cmd.AddCommand(newInternalFWPrintEnvCmd()) + cmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "path to MonoKSConfig yaml") return cmd } diff --git a/clitools/pkg/controller/osupgrade/bootenv.go b/clitools/pkg/controller/osupgrade/bootenv.go new file mode 100644 index 0000000..b632694 --- /dev/null +++ b/clitools/pkg/controller/osupgrade/bootenv.go @@ -0,0 +1,37 @@ +package osupgrade + +import ( + "context" + "fmt" + "os/exec" +) + +type NextBootConfig struct { + Key string + Value string +} + +func SetNextBootEnv(ctx context.Context, cfg NextBootConfig) error { + if cfg.Key == "" { + return fmt.Errorf("boot env key is empty") + } + if cfg.Value == "" { + return fmt.Errorf("boot env value is empty") + } + + cmd := exec.CommandContext( + ctx, + "/proc/self/exe", + "internal", + "fw-setenv", + "--key", cfg.Key, + "--value", cfg.Value, + ) + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("invoke internal fw-setenv: %w: %s", err, string(out)) + } + + return nil +} diff --git a/clitools/pkg/controller/osupgrade/handler.go b/clitools/pkg/controller/osupgrade/handler.go index 57f587e..51a519a 100644 --- a/clitools/pkg/controller/osupgrade/handler.go +++ b/clitools/pkg/controller/osupgrade/handler.go @@ -128,7 +128,22 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients, } klog.Info(result) - // TODO: fw_setenv + if err := SetNextBootEnv(ctx, NextBootConfig{ + Key: "boot_part", + Value: "B", + }); err != nil { + return failProgress(ctx, clients, osup, "set boot env", err) + } + + now = metav1.Now() + osup.Status.LastUpdatedAt = &now + osup.Status.Message = "image applied, verified, and next boot environment updated" + osup.Status.Phase = monov1alpha1.OSUpgradeProgressPhaseRebooting + + osup, err = updateProgressStatus(ctx, clients, osup_gvr, osup) + if err != nil { + return fmt.Errorf("update progress status: %w", err) + } return nil } diff --git a/clitools/pkg/node/agent.go b/clitools/pkg/node/agent.go index 976fb9f..d52d63f 100644 --- a/clitools/pkg/node/agent.go +++ b/clitools/pkg/node/agent.go @@ -273,20 +273,8 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter Value: "/host/opt/monok8s/config/cluster.env", }, { - Name: "HOST_MOUNT_ROOT", - Value: "/host/mnt/control-agent", - }, - { - Name: "HOST_DEV_DIR", - Value: "/host/dev", - }, - { - Name: "HOST_PROC_DIR", - Value: "/host/proc", - }, - { - Name: "HOST_RUN_DIR", - Value: "/host/run", + Name: "FW_ENV_CONFIG_FILE", + Value: "/host/etc/fw_env.config", }, }, SecurityContext: &corev1.SecurityContext{ @@ -295,22 +283,18 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter VolumeMounts: []corev1.VolumeMount{ { Name: "host-dev", - MountPath: "/host/dev", + MountPath: "/dev", + }, + { + Name: "host-etc", + MountPath: "/host/etc", + ReadOnly: true, }, { Name: "host-config", MountPath: "/host/opt/monok8s/config", ReadOnly: true, }, - { - Name: "host-run", - MountPath: "/host/run", - }, - { - Name: "host-proc", - MountPath: "/host/proc", - ReadOnly: true, - }, }, }, }, @@ -324,6 +308,15 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter }, }, }, + { + Name: "host-etc", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc", + Type: hostPathType(corev1.HostPathDirectory), + }, + }, + }, { Name: "host-config", VolumeSource: corev1.VolumeSource{ @@ -333,24 +326,6 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter }, }, }, - { - Name: "host-run", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run", - Type: hostPathType(corev1.HostPathDirectory), - }, - }, - }, - { - Name: "host-proc", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/proc", - Type: hostPathType(corev1.HostPathDirectory), - }, - }, - }, }, }, }, diff --git a/clitools/pkg/node/network.go b/clitools/pkg/node/network.go index b3ee032..7b45b57 100644 --- a/clitools/pkg/node/network.go +++ b/clitools/pkg/node/network.go @@ -7,8 +7,8 @@ import ( "os" "strings" - "k8s.io/klog/v2" system "example.com/monok8s/pkg/system" + "k8s.io/klog/v2" ) type NetworkConfig struct { diff --git a/docs/coredns.md b/docs/coredns.md new file mode 100644 index 0000000..c3155b3 --- /dev/null +++ b/docs/coredns.md @@ -0,0 +1,198 @@ +## About /etc/crio/crio.conf.d/11-coredns.conf + +CoreDNS fails with permission denied. Probably because of readonly RootFS. +The current solution is to just allow port 20 up to bind without root + +well this is just brute force allowing any ports 20 < to be yeah go ahead +``` +[crio.runtime] +default_sysctls = [ + "net.ipv4.ip_unprivileged_port_start=20" +] +``` + +CoreDNS wants 53. And users may want to have ssh servers + +So just allow all the ports! + + +## Diagnostics + +I'll just put some of my findings in here. + +Catching effective cap being 0x0000 (this is the issue?) +``` +kubectl rollout restart deployment -n kube-system coredns +while true; do + PID=$(pgrep coredns) + if [ ! -z "$PID" ]; then + echo "Found CoreDNS PID: $PID" + grep Cap /proc/$PID/status + break + fi +done + +CapInh: 0000000000000000 +CapPrm: 0000000000000000 +CapEff: 0000000000000000 +CapBnd: 0000000000000400 +CapAmb: 0000000000000000 +``` + +# find /var/lib/containers/storage/overlay-containers -name config.json | xargs grep "noNewPrivileges" + +Current kernel config +``` +gunzip -c /proc/config.gz | grep -e SECURITY -e LSM -e SECCOMP +CONFIG_HAVE_ARCH_SECCOMP=y +CONFIG_HAVE_ARCH_SECCOMP_FILTER=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +# CONFIG_SECCOMP_CACHE_DEBUG is not set +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_IIO_ST_LSM6DSX_I2C=m +CONFIG_IIO_ST_LSM6DSX_SPI=m +CONFIG_IIO_ST_LSM6DSX_I3C=m +# CONFIG_IIO_ST_LSM9DS0 is not set +CONFIG_EXT4_FS_SECURITY=y +CONFIG_UBIFS_FS_SECURITY=y +CONFIG_NFS_V4_SECURITY_LABEL=y +# CONFIG_9P_FS_SECURITY is not set +# CONFIG_SECURITY_DMESG_RESTRICT is not set +CONFIG_SECURITY=y +CONFIG_HAS_SECURITY_AUDIT=y +CONFIG_SECURITYFS=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_SECURITY_PATH=y +# CONFIG_SECURITY_SELINUX is not set +# CONFIG_SECURITY_SMACK is not set +# CONFIG_SECURITY_TOMOYO is not set +# CONFIG_SECURITY_APPARMOR is not set +# CONFIG_SECURITY_LOADPIN is not set +# CONFIG_SECURITY_YAMA is not set +# CONFIG_SECURITY_SAFESETID is not set +# CONFIG_SECURITY_LOCKDOWN_LSM is not set +# CONFIG_SECURITY_LANDLOCK is not set +# CONFIG_SECURITY_IPE is not set +CONFIG_DEFAULT_SECURITY_DAC=y +CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,ipe,bpf" +``` + +`CONFIG_LSM` does not need `capability`. It works when testing CAP_NET_BIND manually. +``` +# capsh --keep=1 --user=nobody --inh=cap_net_bind_service --addamb=cap_net_bind_service -- -c "grep Cap /proc/self/status" +CapInh: 0000000000000400 +CapPrm: 0000000000000400 +CapEff: 0000000000000400 +CapBnd: 000001ffffffffff +CapAmb: 0000000000000400 +``` + +And this also aligns with debian's `CONFIG_LSM`. They don't have `capability` hardcoded in. + +Debian's working kernel config +``` +grep -e SECURITY -e LSM -e SECCOMP /boot/config-$(uname -r) +CONFIG_BPF_LSM=y +# CONFIG_NFIT_SECURITY_DEBUG is not set +CONFIG_HAVE_ARCH_SECCOMP=y +CONFIG_HAVE_ARCH_SECCOMP_FILTER=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +# CONFIG_SECCOMP_CACHE_DEBUG is not set +CONFIG_IP_NF_SECURITY=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_IIO_ST_LSM6DSX_I2C=m +CONFIG_IIO_ST_LSM6DSX_SPI=m +# CONFIG_IIO_ST_LSM9DS0 is not set +CONFIG_EXT4_FS_SECURITY=y +CONFIG_REISERFS_FS_SECURITY=y +CONFIG_JFS_SECURITY=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_JFFS2_FS_SECURITY=y +CONFIG_UBIFS_FS_SECURITY=y +CONFIG_EROFS_FS_SECURITY=y +CONFIG_NFS_V4_SECURITY_LABEL=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +# CONFIG_CEPH_FS_SECURITY_LABEL is not set +CONFIG_9P_FS_SECURITY=y +CONFIG_SECURITY_DMESG_RESTRICT=y +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y +CONFIG_SECURITY=y +CONFIG_SECURITYFS=y +CONFIG_SECURITY_NETWORK=y +# CONFIG_SECURITY_INFINIBAND is not set +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_SECURITY_PATH=y +CONFIG_LSM_MMAP_MIN_ADDR=65536 +CONFIG_SECURITY_SELINUX=y +# CONFIG_SECURITY_SELINUX_BOOTPARAM is not set +# CONFIG_SECURITY_SELINUX_DISABLE is not set +CONFIG_SECURITY_SELINUX_DEVELOP=y +CONFIG_SECURITY_SELINUX_AVC_STATS=y +CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=0 +CONFIG_SECURITY_SELINUX_SIDTAB_HASH_BITS=9 +CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE=256 +# CONFIG_SECURITY_SMACK is not set +CONFIG_SECURITY_TOMOYO=y +CONFIG_SECURITY_TOMOYO_MAX_ACCEPT_ENTRY=2048 +CONFIG_SECURITY_TOMOYO_MAX_AUDIT_LOG=1024 +# CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER is not set +CONFIG_SECURITY_TOMOYO_POLICY_LOADER="/sbin/tomoyo-init" +CONFIG_SECURITY_TOMOYO_ACTIVATION_TRIGGER="/sbin/init" +# CONFIG_SECURITY_TOMOYO_INSECURE_BUILTIN_SETTING is not set +CONFIG_SECURITY_APPARMOR=y +# CONFIG_SECURITY_APPARMOR_DEBUG is not set +CONFIG_SECURITY_APPARMOR_INTROSPECT_POLICY=y +CONFIG_SECURITY_APPARMOR_HASH=y +CONFIG_SECURITY_APPARMOR_HASH_DEFAULT=y +CONFIG_SECURITY_APPARMOR_EXPORT_BINARY=y +CONFIG_SECURITY_APPARMOR_PARANOID_LOAD=y +# CONFIG_SECURITY_LOADPIN is not set +CONFIG_SECURITY_YAMA=y +# CONFIG_SECURITY_SAFESETID is not set +CONFIG_SECURITY_LOCKDOWN_LSM=y +CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_IMA_LSM_RULES=y +# CONFIG_DEFAULT_SECURITY_SELINUX is not set +# CONFIG_DEFAULT_SECURITY_TOMOYO is not set +CONFIG_DEFAULT_SECURITY_APPARMOR=y +# CONFIG_DEFAULT_SECURITY_DAC is not set +CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,integrity,apparmor,selinux,smack,tomoyo,bpf" +``` + +Debian's CoreDNS cap +``` +# grep Cap /proc/1911/status +CapInh: 0000000000000000 +CapPrm: 0000000000000400 +CapEff: 0000000000000400 +CapBnd: 0000000000000400 +CapAmb: 0000000000000000 +``` + +Debian's CRIO config +``` +# cat /etc/crio/crio.conf.d/10-crio.conf +[crio.image] +signature_policy = "/etc/crio/policy.json" + +[crio.runtime] +default_runtime = "crun" + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/libexec/crio/crun" +runtime_root = "/run/crun" +monitor_path = "/usr/libexec/crio/conmon" +allowed_annotations = [ + "io.containers.trace-syscall", +] + +[crio.runtime.runtimes.runc] +runtime_path = "/usr/libexec/crio/runc" +runtime_root = "/run/runc" +monitor_path = "/usr/libexec/crio/conmon" +``` diff --git a/docs/uboot.md b/docs/uboot.md index 7b29601..e393d43 100644 --- a/docs/uboot.md +++ b/docs/uboot.md @@ -1,4 +1,4 @@ -Booting release image with bootusb +## Booting release image with bootusb - Use this if you are flashing image from macos/flashusb.sh ``` setenv bootusb ' @@ -18,8 +18,16 @@ setenv bootusb 'usb start; ext4load usb 0:2 ${mks_loadaddr} /boot/kernel.itb; bo run bootusb ``` +## Run fw_printenv and fw_setenv from kubectl +``` +# Avoid using daemonset/control-agent if you have multiple nodes +kubectl exec -n kube-system control-agent-abcdef1 -- /ctl internal fw-setenv --key foo --value bar -Original uboot env from mono +# fw_printenv +kubectl exec -n kube-system ds/control-agent -- /ctl internal fw-printenv --key foo +``` + +## Original uboot env from mono ``` arch=arm baudrate=115200 diff --git a/kernel-extra.config b/kernel-extra.config index efd4499..833cb56 100644 --- a/kernel-extra.config +++ b/kernel-extra.config @@ -293,6 +293,14 @@ CONFIG_SECCOMP_FILTER=y # CONFIG_SECURITY=y # CONFIG_SECURITYFS=y +CONFIG_SECURITY_NETWORK=y +# enables Security Module (LSM) hooks for network operations. CoreDNS needs this + +CONFIG_SECURITY_PATH=y +# Recommended for container isolation + +CONFIG_SECURITY_NETWORK_XFRM=y + ############################################################################### # Misc userspace/container compatibility diff --git a/makefile b/makefile index 48a64ae..911a542 100644 --- a/makefile +++ b/makefile @@ -231,20 +231,6 @@ $(RELEASE_IMAGE): $(RELEASE_DEPS) | $(OUT_DIR) test -f $@ -check-functions: - @echo "Checking function calls vs definitions..." - @awk '/^main\(\)/ { in_main=1; next } \ - in_main && /^\}/ { in_main=0 } \ - in_main && /^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*$$/ { \ - gsub(/^[ \t]+/, "", $$0); \ - print $$0 \ - }' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \ - | sort -u > /tmp/called.txt - @grep -E '^[a-zA-Z_][a-zA-Z0-9_]*\(\)' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \ - | sed 's/().*//' \ - | sort -u > /tmp/defined.txt - @echo "Missing functions:" - @comm -23 /tmp/called.txt /tmp/defined.txt || true # ---- config targets ------------------------------------------------------------