Reduce the number of bootenv vars

This commit is contained in:
2026-04-06 02:20:41 +08:00
parent f8db036a5f
commit 50d9440e0a
14 changed files with 455 additions and 93 deletions

View File

@@ -10,7 +10,7 @@ https://docs.mono.si/gateway-development-kit/getting-started
## IMPORTANT NOTES
* This is not your everyday linux image! It is best suited for users that is already familiar
with k8s. For first-timers, you may want to try the default config that gives you a ready-to-used
with k8s. For first-timers, you may want to try the default config that gives you a ready-to-use
cluster then get yourself started from there
* The 3 RJ45 ports are label in eth1, eth2, eth0 respectively by the kernel (left to right)

View File

@@ -5,8 +5,8 @@ TAG=dev
# The Linux kernel, from NXP
NXP_VERSION=lf-6.18.2-1.0.0
CRIO_VERSION=cri-o.arm64.v1.35.2
KUBE_VERSION=v1.34.1
CRIO_VERSION=cri-o.arm64.v1.33.3
KUBE_VERSION=v1.33.3
# Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one
# We need fsl-ls1046a-rdb-sdk.dtb here

View File

@@ -3,6 +3,7 @@ package osupgrade
import (
"context"
"fmt"
"os"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
@@ -12,6 +13,7 @@ import (
"example.com/monok8s/pkg/catalog"
"example.com/monok8s/pkg/controller/osimage"
"example.com/monok8s/pkg/kube"
"example.com/monok8s/pkg/node/uboot"
)
func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
@@ -89,7 +91,7 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
imageOptions := osimage.ApplyOptions{
URL: first.URL,
TargetPath: "./out/flash.img",
TargetPath: "/dev/sda?",
ExpectedRawSHA256: imageSHA,
ExpectedRawSize: first.Size,
BufferSize: 6 * 1024 * 1024,
@@ -128,10 +130,9 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
}
klog.Info(result)
if err := SetNextBootEnv(ctx, NextBootConfig{
Key: "boot_part",
Value: "B",
}); err != nil {
cfgPath := os.Getenv("FW_ENV_CONFIG_FILE")
if err := uboot.ConfigureNextBoot(ctx, cfgPath); err != nil {
return failProgress(ctx, clients, osup, "set boot env", err)
}
@@ -145,5 +146,8 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
return fmt.Errorf("update progress status: %w", err)
}
// TODO: Drain the node here
// TODO: Issue Reboot
return nil
}

View File

@@ -118,6 +118,16 @@ func applyControlAgentClusterRole(ctx context.Context, kubeClient kubernetes.Int
Resources: []string{"osupgrades/status"},
Verbs: []string{"get", "patch", "update"},
},
{
APIGroups: []string{monov1alpha1.Group},
Resources: []string{"osupgradeprogresses"},
Verbs: []string{"get", "list", "watch", "create", "patch", "update"},
},
{
APIGroups: []string{monov1alpha1.Group},
Resources: []string{"osupgradeprogresses/status"},
Verbs: []string{"get", "list", "watch", "create", "patch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"nodes"},

View File

@@ -21,10 +21,37 @@ func ConfigureABBoot(ctx context.Context, nctx *node.NodeContext) error {
return writer.EnsureBootEnv(ctx, BootEnvConfig{
BootSource: "usb",
BootPart: "A",
BootDisk: 0,
RootfsAPartNum: 2,
RootfsBPartNum: 3,
DataPartNum: 4,
LinuxRootPrefix: "/dev/sda",
})
}
// This is called by the agent controller/osupgrade/handler.go
func ConfigureNextBoot(ctx context.Context, fwEnvCfgPath string) error {
exePath, err := os.Executable()
if err != nil {
return fmt.Errorf("get current executable path: %w", err)
}
writer := NewFWEnvWriter(fwEnvCfgPath, exePath)
currBootPart, err := writer.GetEnv(ctx, "boot_part")
if err != nil {
return fmt.Errorf("get boot_part: %w", err)
}
next := "A"
if currBootPart == "A" {
next = "B"
}
currBootSource, err := writer.GetEnv(ctx, "boot_source")
if err != nil {
return fmt.Errorf("get boot_source: %w", err)
}
return writer.EnsureBootEnv(ctx, BootEnvConfig{
BootSource: currBootSource,
BootPart: next,
})
}

View File

@@ -0,0 +1,12 @@
package uboot
import (
"fmt"
"testing"
)
func TestBootCmd(_ *testing.T) {
cfg := BootEnvConfig{}
fmt.Println(cfg.bootCmdOrDefault())
}

View File

@@ -18,22 +18,6 @@ func (c BootEnvConfig) Validate() error {
return fmt.Errorf("invalid boot part %q", c.BootPart)
}
if c.BootDisk < 0 {
return fmt.Errorf("invalid boot disk %d", c.BootDisk)
}
if c.RootfsAPartNum <= 0 {
return fmt.Errorf("invalid rootfs A part %d", c.RootfsAPartNum)
}
if c.RootfsBPartNum <= 0 {
return fmt.Errorf("invalid rootfs B part %d", c.RootfsBPartNum)
}
if c.DataPartNum <= 0 {
return fmt.Errorf("invalid data part %d", c.DataPartNum)
}
if strings.TrimSpace(c.LinuxRootPrefix) == "" {
return fmt.Errorf("linux root prefix is required")
}
return nil
}

View File

@@ -126,11 +126,6 @@ func (w *FWEnvWriter) EnsureBootEnv(ctx context.Context, cfg BootEnvConfig) erro
{"bootcmd", cfg.bootCmdOrDefault()},
{"boot_source", string(cfg.BootSource)},
{"boot_part", string(cfg.BootPart)},
{"boot_disk", fmt.Sprintf("%d", cfg.BootDisk)},
{"rootfs_a_partnum", fmt.Sprintf("%d", cfg.RootfsAPartNum)},
{"rootfs_b_partnum", fmt.Sprintf("%d", cfg.RootfsBPartNum)},
{"data_partnum", fmt.Sprintf("%d", cfg.DataPartNum)},
{"linux_root_prefix", cfg.LinuxRootPrefix},
}
for _, kv := range envs {

View File

@@ -3,6 +3,7 @@ package uboot
import (
"bytes"
"context"
"errors"
"fmt"
"k8s.io/klog/v2"
"os"
@@ -155,7 +156,7 @@ func ConfigureUBootCommands(ctx context.Context, n *node.NodeContext) error {
b.WriteString(trimOutput(a.output, 600))
b.WriteString("\n")
}
return fmt.Errorf(b.String())
return errors.New(b.String())
}
klog.Infof("Using: %s", best.config)

View File

@@ -14,11 +14,6 @@ const (
type BootEnvConfig struct {
BootSource string // usb or emmc
BootPart string // A or B
BootDisk int
RootfsAPartNum int
RootfsBPartNum int
DataPartNum int
LinuxRootPrefix string // /dev/sda or /dev/mmcblk0p
BootCmd string // optional; defaults to DefaultBootCmd
}
@@ -29,32 +24,52 @@ type FWEnvWriter struct {
}
const defaultBootCmdTemplate = `
setenv kernel_addr_r 0xa0000000;
if usb start; then
if usb dev 0; then
echo "Trying generic USB boot...";
# Prefer extlinux-style boot
if test -e usb 0:1 /boot/extlinux/extlinux.conf; then
echo "Found extlinux config on USB";
sysboot usb 0:1 any ${kernel_addr_r} /boot/extlinux/extlinux.conf;
fi;
# Fallback: U-Boot script image
if test -e usb 0:1 /boot/boot.scr; then
echo "Found boot.scr on USB";
if load usb 0:1 ${kernel_addr_r} /boot/boot.scr; then
source ${kernel_addr_r};
fi;
fi;
fi;
fi;
if test "${boot_source}" = "usb"; then
usb start || exit;
setenv boot_iface usb;
usb dev 0 || exit;
elif test "${boot_source}" = "emmc"; then
mmc dev ${boot_disk} || exit;
mmc dev 0 || exit;
setenv boot_iface mmc;
else
echo "unsupported boot_source: ${boot_source}";
exit;
fi;
setenv kernel_addr_r 0xa0000000;
if test "${boot_part}" = "A"; then
setenv rootpart ${rootfs_a_partnum};
setenv rootpart 2;
elif test "${boot_part}" = "B"; then
setenv rootpart ${rootfs_b_partnum};
setenv rootpart 3;
else
echo "unsupported boot_part: ${boot_part}";
exit;
fi;
setenv bootdev ${boot_disk}:${rootpart};
setenv rootdev ${linux_root_prefix}${rootpart};
setenv datadev ${linux_root_prefix}${data_partnum};
setenv bootdev 0:${rootpart};
setenv rootdev ${boot_source}:${rootpart};
setenv bootargs "${bootargs_console} root=${rootdev} data=${datadev} rw rootwait rootfstype=ext4";
setenv bootargs "${bootargs_console} root=${rootdev} bootpart=${boot_part} rw rootwait rootfstype=ext4";
ext4load ${boot_iface} ${bootdev} ${kernel_addr_r} /boot/kernel.itb && bootm ${kernel_addr_r};
`

View File

@@ -33,6 +33,18 @@ MKS_INIT_CONTROL_PLANE=yes
# OSUpgrade agent
MKS_ENABLE_CONTROL_AGENT=yes
# Boot configs
# usb, emmc
MKS_BOOT_SOURCE="usb"
MKS_BOOT_DEV_PREFIX="/dev/sda"
# Depends on your full disk image. This is the default
MKS_BOOT_PART=A
MKS_BOOT_DISK=0
MKS_ROOTFS_PART_A=2
MKS_ROOTFS_PART_B=3
MKS_ROOTFS_PART_D=4
MKS_API_SERVER_ENDPOINT=
MKS_BOOTSTRAP_TOKEN=
MKS_DISCOVERY_TOKEN_CA_CERT_HASH=

View File

@@ -1,10 +1,12 @@
## Upgrade process
We use a CRD with an agent to handle this. Our versions follows upstream's.
We use an agent to watch the OSUpgrade CRD to handle this. Our image versions follows upstream.
To issue an upgrade. Simply use
kubectl apply -f upgrade.yaml
Example yaml
```yaml
apiVersion: monok8s.io/v1alpha1
kind: OSUpgrade
@@ -12,8 +14,6 @@ metadata:
name: "my-ugrade-2"
spec:
version: "v1.35.3"
imageURL: "https://updates.example.com/monok8s-1.2.3.img.zst"
checksum: "sha256:..."
nodeSelector: {}
catalog:
inline: |
@@ -39,7 +39,7 @@ spec:
- v1.34.0
```
catalog accepts URL or ConfigMap
catalog also accepts URL or ConfigMap
```yaml
catalog:
URL: https://example.com/images.yaml
@@ -48,6 +48,9 @@ catalog:
ConfigMap: images-cm
```
※ ConfigMap requires additional RBAC permission which is not by default. You edit the
control-agent ClusterRole and add `cnofigmaps: get` to allow this.
Contents should look like this
```yaml
stable: v1.35.1

View File

@@ -42,6 +42,168 @@ wait_for_path() {
done
}
get_cmdline_arg() {
key="$1"
for arg in $(cat /proc/cmdline); do
case "$arg" in
"$key"=*)
echo "${arg#"$key"=}"
return 0
;;
esac
done
return 1
}
# Read KEY=VALUE pairs from /sys/class/block/*/uevent without spawning grep/cut.
get_uevent_value() {
file="$1"
want_key="$2"
[ -f "$file" ] || return 1
while IFS='=' read -r k v; do
[ "$k" = "$want_key" ] && {
echo "$v"
return 0
}
done < "$file"
return 1
}
# Return the /dev/<partition> path for the first partition whose GPT PARTNAME matches.
find_first_part_by_partname() {
want_label="$1"
for p in /sys/class/block/*; do
[ -f "$p/partition" ] || continue
partname="$(get_uevent_value "$p/uevent" PARTNAME || true)"
[ "$partname" = "$want_label" ] || continue
devname="$(basename "$p")"
echo "/dev/$devname"
return 0
done
return 1
}
wait_for_partnames() {
timeout="${1:-3}"
shift
i=0
while [ "$i" -lt "$timeout" ]; do
all_found=1
for name in "$@"; do
if ! find_first_part_by_partname "$name" >/dev/null; then
all_found=0
break
fi
done
[ "$all_found" -eq 1 ] && return 0
sleep 1
i=$((i + 1))
log "Still waiting for $@ to populate($i)"
done
return 1
}
find_part_by_partuuid() {
want="$1"
for p in /sys/class/block/*; do
[ -f "$p/partition" ] || continue
partuuid="$(get_uevent_value "$p/uevent" PARTUUID || true)"
[ "$partuuid" = "$want" ] || continue
echo "/dev/$(basename "$p")"
return 0
done
return 1
}
# Return the parent disk name for a partition device name.
# Examples:
# sda2 -> sda
# mmcblk0p2 -> mmcblk0
parent_disk_name_for_part() {
part_devname="$1"
real="$(readlink -f "/sys/class/block/$part_devname")" || return 1
parent="$(basename "$(dirname "$real")")" || return 1
echo "$parent"
return 0
}
# Find a sibling partition on the same disk by GPT PARTNAME.
find_sibling_part_on_same_disk() {
part_path="$1"
want_label="$2"
part_devname="$(basename "$part_path")"
disk_devname="$(parent_disk_name_for_part "$part_devname")" || return 1
for p in /sys/class/block/"$disk_devname"*; do
[ -f "$p/partition" ] || continue
partname="$(get_uevent_value "$p/uevent" PARTNAME || true)"
[ "$partname" = "$want_label" ] || continue
echo "/dev/$(basename "$p")"
return 0
done
return 1
}
# Resolve preferred root device from sysfs.
# Prefer PARTUUID first, then optionally filesystem UUID if explicitly provided.
resolve_preferred_root() {
pref_root="$1"
[ -n "$pref_root" ] || return 1
find_part_by_partuuid "$pref_root"
}
# Decide which root PARTNAME we want for the requested slot.
# Keep a compatibility fallback for legacy "rootfs" as slot A.
wanted_root_labels_for_slot() {
slot="$1"
case "$slot" in
B|b)
echo "rootfsB"
;;
*)
# Try modern rootfsA first, then legacy rootfs
echo "rootfsA rootfs"
;;
esac
}
find_fallback_root_for_slot() {
slot="$1"
for label in $(wanted_root_labels_for_slot "$slot"); do
dev="$(find_first_part_by_partname "$label" || true)"
if [ -n "$dev" ]; then
echo "$dev"
return 0
fi
done
return 1
}
mkdir -p /dev /proc /sys /run
mount_or_panic -t devtmpfs devtmpfs /dev
mount_or_panic -t proc proc /proc
@@ -62,45 +224,56 @@ log "Booting kernel took $(cut -d' ' -f1 /proc/uptime) seconds."
. /etc/build-info || panic "failed to source /etc/build-info"
ROOT_DEV=""
DATA_DEV=""
wait_for_partnames 30 rootfsA rootfsB data || panic "failed to wait for fs"
for arg in $(cat /proc/cmdline); do
case "$arg" in
root=*)
ROOT_DEV="${arg#root=}"
;;
data=*)
DATA_DEV="${arg#data=}"
;;
esac
done
ROOT_CMD="$(get_cmdline_arg root || true)"
BOOT_PART="$(get_cmdline_arg bootpart || true)"
PREFERRED_PARTUUID="$(get_cmdline_arg pref_root || true)"
[ -n "$ROOT_DEV" ] || {
log "No root= specified in cmdline"
exec sh
}
ROOT_DEV="$(resolve_preferred_root "$PREFERRED_PARTUUID" || true)"
if [ -n "$ROOT_DEV" ]; then
log "Using preferred root device: $ROOT_DEV"
fi
[ -n "$DATA_DEV" ] || {
log "No data= specified in cmdline"
exec sh
}
if [ -z "$ROOT_DEV" ]; then
ROOT_DEV="$(find_fallback_root_for_slot "$BOOT_PART" || true)"
if [ -n "$ROOT_DEV" ]; then
log "Preferred root not found. Falling back to first valid root device: $ROOT_DEV"
fi
fi
[ -n "$ROOT_DEV" ] || panic "no usable root device found"
DATA_DEV="$(find_sibling_part_on_same_disk "$ROOT_DEV" data || true)"
[ -n "$DATA_DEV" ] || panic "no data partition found on same disk as $ROOT_DEV"
wait_for_path "$ROOT_DEV"
wait_for_path "$DATA_DEV"
e2fsck -p "$DATA_DEV" || {
echo "Auto fsck failed, forcing repair"
log "Auto fsck failed, forcing repair"
e2fsck -y "$DATA_DEV" || panic "fsck failed on $DATA_DEV"
}
mkdir -p /newroot
mkdir -p /newroot/data
mkdir -p /newroot/var
mount_retry "$ROOT_DEV" /newroot ext4 ro
mount_retry "$DATA_DEV" /newroot/data ext4 rw
mkdir -p /newroot/data/var
mkdir -p /newroot/data/etc-overlay/upper
mkdir -p /newroot/data/etc-overlay/work
mount_or_panic --bind /newroot/data/var /newroot/var
# BusyBox mount just needs a normal -o option string here.
# The important bit is that overlayfs itself requires lowerdir/upperdir/workdir,
# and workdir must live on the same filesystem as upperdir.
mount_or_panic -t overlay overlay \
-o lowerdir=/newroot/etc,upperdir=/newroot/data/etc-overlay/upper,workdir=/newroot/data/etc-overlay/work \
-o "lowerdir=/newroot/etc,upperdir=/newroot/data/etc-overlay/upper,workdir=/newroot/data/etc-overlay/work" \
/newroot/etc
mount_or_panic --move /dev /newroot/dev
@@ -108,7 +281,7 @@ mount_or_panic --move /proc /newroot/proc
mount_or_panic --move /sys /newroot/sys
mount_or_panic --move /run /newroot/run
log "Switching root to $ROOT_DEV"
log "Switching root to $ROOT_DEV (data: $DATA_DEV, slot: $BOOT_PART)"
exec switch_root /newroot /sbin/init
panic "switch_root returned unexpectedly"

View File

@@ -2,10 +2,9 @@
set -eu
### User-configurable vars
IMG="./out/monok8s-dev.img.gz"
DEFAULT_DISK="/dev/disk7" # whole disk, not a partition
IMG_DIR="./out"
CONFIG_PART_SUFFIX="s1" # config partition after flashing
ENV_SEARCH_DIR="./out" # optional; leave empty to use image dir
ENV_SEARCH_DIR="./out" # optional; leave empty to use selected image dir
MNT="/tmp/monok8s-usb-config.$$"
@@ -59,6 +58,131 @@ wait_for_mountable_partition() {
return 1
}
select_image() {
dir="$1"
[ -d "$dir" ] || die "Image directory not found: $dir"
OLD_IFS="${IFS}"
IFS='
'
set -- $(find "$dir" -maxdepth 1 -type f -name '*.img.gz' | sort)
IFS="${OLD_IFS}"
[ "$#" -gt 0 ] || die "No .img.gz images found in $dir"
echo "Available images:"
n=1
for img in "$@"; do
size="$(stat -f%z "$img" 2>/dev/null || echo "?")"
echo " [$n] $(basename "$img") (${size} bytes)"
n=$((n + 1))
done
echo
printf "Select image number: "
read -r choice
case "$choice" in
''|*[!0-9]*)
die "Invalid selection: $choice"
;;
esac
n=1
for img in "$@"; do
if [ "$n" -eq "$choice" ]; then
SELECTED_IMG="$img"
return 0
fi
n=$((n + 1))
done
die "Selection out of range: $choice"
}
list_candidate_disks() {
found=0
for dev in /dev/disk*; do
base="$(basename "$dev")"
case "$base" in
disk[0-9])
;;
*)
continue
;;
esac
info="$(diskutil list "$dev" 2>/dev/null || true)"
[ -n "$info" ] || continue
# Skip Apple disks
echo "$info" | grep -q "APFS" && continue
info="$(diskutil info "$dev" 2>/dev/null || true)"
found=1
media_name="$(echo "$info" | sed -n 's/^ *Device \/ Media Name: *//p' | head -n1)"
protocol="$(echo "$info" | sed -n 's/^ *Protocol: *//p' | head -n1)"
size="$(echo "$info" | sed -n 's/^ *Disk Size: *//p' | head -n1)"
echo "$dev|$media_name|$protocol|$size"
done
[ "$found" -eq 1 ] || return 1
return 0
}
select_disk() {
candidates="$(list_candidate_disks || true)"
if [ -z "$candidates" ]; then
warn "No obvious external candidate disks found."
printf "Enter target disk manually (example: /dev/disk7): "
read -r TARGET_DISK
[ -n "$TARGET_DISK" ] || die "No disk entered"
return 0
fi
echo "Candidate target disks:"
n=1
echo "$candidates" | while IFS='|' read -r dev media protocol size; do
echo " [$n] $dev - $media - $protocol - $size"
n=$((n + 1))
done
echo
printf "Select disk number (or type a full /dev/diskN path): "
read -r choice
case "$choice" in
/dev/disk[0-9]*)
TARGET_DISK="$choice"
return 0
;;
''|*[!0-9]*)
die "Invalid selection: $choice"
;;
esac
n=1
OLD_IFS="${IFS}"
IFS='
'
for line in $candidates; do
if [ "$n" -eq "$choice" ]; then
TARGET_DISK="$(echo "$line" | cut -d'|' -f1)"
IFS="${OLD_IFS}"
return 0
fi
n=$((n + 1))
done
IFS="${OLD_IFS}"
die "Selection out of range: $choice"
}
### Sanity checks
require_cmd diskutil
require_cmd gunzip
@@ -70,24 +194,26 @@ require_cmd cp
require_cmd sed
require_cmd stat
require_cmd pv
require_cmd cut
require_cmd grep
require_cmd head
printf "disk ($DEFAULT_DISK): "
read -r TARGET_DISK
select_image "$IMG_DIR"
IMG="$SELECTED_IMG"
if [ -z "$TARGET_DISK" ]; then
TARGET_DISK="$DEFAULT_DISK"
fi
select_disk
TARGET_RAW_DISK="/dev/r$(basename "$TARGET_DISK")"
TARGET_BASENAME="$(basename "$TARGET_DISK")"
### Derived vars
TARGET_CONFIG_PART="/dev/${TARGET_BASENAME}${CONFIG_PART_SUFFIX}"
IMG_DIR="$(cd "$(dirname "$IMG")" && pwd)"
SEARCH_DIR="${ENV_SEARCH_DIR:-$IMG_DIR}"
IMG_SIZE_BYTES="$(stat -f%z "$IMG")"
IMG_DIR_ABS="$(cd "$(dirname "$IMG")" && pwd)"
SEARCH_DIR="${ENV_SEARCH_DIR:-$IMG_DIR_ABS}"
[ -f "$IMG" ] || die "Image not found: $IMG"
IMG_SIZE_BYTES="$(stat -f%z "$IMG")"
[ -b "$TARGET_DISK" ] || die "Target disk not found: $TARGET_DISK"
[ "$(id -u)" -eq 0 ] || die "Run this script with sudo"
@@ -122,7 +248,7 @@ diskutil unmountDisk force "$TARGET_DISK" || die "Failed to unmount $TARGET_DISK
log "Flashing image with progress..."
pv -s "$IMG_SIZE_BYTES" "$IMG" \
| gunzip \
| dd of="$TARGET_RAW_DISK" bs=1m
| dd of="$TARGET_RAW_DISK" bs=4m
log "Syncing writes..."
sync