Initial upgrade flow for control-plane

README.md

@@ -16,7 +16,7 @@ https://docs.mono.si/gateway-development-kit/getting-started
 
  * USE AT YOUR OWN RISKS. I leverage ChatGPT heavily for this.
 
-### Table of Contents
+## Table of Contents
 1. Flashing
    - [USB](docs/flashing-usb.md)
    - [Over network (tftp)](docs/flashing-network.md)

alpine/rootfs-extra/etc/containers/storage.conf

@@ -1,9 +1 @@
-[storage]
-driver = "overlay"
-runroot = "/run/containers/storage"
-graphroot = "/var/lib/containers/storage"
-
-[storage.options]
-additionalimagestores = [
-  "/usr/lib/monok8s/imagestore"
-]
+# Generated file. DO NOT MODIFY.

build.env

@@ -5,8 +5,8 @@ TAG=dev
 
 # The Linux kernel, from NXP
 NXP_VERSION=lf-6.18.2-1.0.0
-CRIO_VERSION=cri-o.arm64.v1.33.3
-KUBE_VERSION=v1.33.3
+CRIO_VERSION=cri-o.arm64.v1.33.10
+KUBE_VERSION=v1.33.10
 
 # Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one
 # We need fsl-ls1046a-rdb-sdk.dtb here

clitools/makefile

@@ -3,7 +3,7 @@ VERSION ?= dev
 UBOOT_VERSION=v2026.01
 
 # Target kube version
-KUBE_VERSION ?= v1.35.0
+KUBE_VERSION ?= v1.33.3
 
 GIT_REV := $(shell git rev-parse HEAD)
 

clitools/pkg/apis/monok8s/v1alpha1/groupversion_info.go

@@ -11,13 +11,17 @@ import (
 var (
 	Group      = "monok8s.io"
 	Version    = "v1alpha1"
+	APIVersion = "monok8s.io/v1alpha1"
+
+	AltPartDeviceLink = "/dev/mksaltpart"
+	Annotation        = "monok8s.io/annotation"
+	BootStateFile     = "/run/monok8s/boot-state.env"
+	CatalogURL        = "https://example.com/monok8s.io/v1alpha1/catalog.yaml"
+	ControlAgentKey   = "monok8s.io/control-agent"
+	EnvConfigDir      = "/opt/monok8s/config"
+	Label             = "monok8s.io/label"
 	MonoKSConfigCRD   = "monoksconfigs.monok8s.io"
 	OSUpgradeCRD      = "osupgrades.monok8s.io"
-	APIVersion      = "monok8s.io/v1alpha1"
-	Label           = "monok8s.io/label"
-	Annotation      = "monok8s.io/annotation"
-	ControlAgentKey = "monok8s.io/control-agent"
-	CatalogURL      = "https://example.com/monok8s.io/v1alpha1/catalog.yaml"
 )
 
 var (

clitools/pkg/bootstrap/registry.go

@@ -27,21 +27,25 @@ func NewRegistry(ctx *node.NodeContext) *Registry {
 			"ApplyLocalNodeMetadataIfPossible":       node.ApplyLocalNodeMetadataIfPossible,
 			"CheckForVersionSkew":                    node.CheckForVersionSkew,
 			"ClassifyBootstrapAction":                node.ClassifyBootstrapAction,
+			"ConfigureABBoot":                        uboot.ConfigureABBoot,
 			"ConfigureDNS":                           node.ConfigureDNS(netCfg),
 			"ConfigureDefaultCNI":                    node.ConfigureDefaultCNI,
 			"ConfigureHostname":                      node.ConfigureHostname(netCfg),
 			"ConfigureMgmtInterface":                 node.ConfigureMgmtInterface(netCfg),
-			"ConfigureABBoot":                        uboot.ConfigureABBoot,
 			"ConfigureUBootCommands":                 uboot.ConfigureUBootCommands,
 			"DetectLocalClusterState":                node.DetectLocalClusterState,
+			"EngageControlGate":                      node.EngageControlGate,
 			"EnsureIPForward":                        node.EnsureIPForward,
+			"MountAltImageStore":                     node.MountAltImageStore,
 			"ReconcileControlPlane":                  node.ReconcileControlPlane,
 			"ReconcileWorker":                        node.ReconcileWorker,
+			"ReleaseControlGate":                     node.ReleaseControlGate,
 			"RunKubeadmInit":                         node.RunKubeadmInit,
 			"RunKubeadmJoin":                         node.RunKubeadmJoin,
 			"RunKubeadmUpgradeApply":                 node.RunKubeadmUpgradeApply,
 			"RunKubeadmUpgradeNode":                  node.RunKubeadmUpgradeNode,
 			"StartCRIO":                              node.StartCRIO,
+			"UnmountAltImageStore":                   node.UnmountAltImageStore,
 			"ValidateNodeIPAndAPIServerReachability": node.ValidateNodeIPAndAPIServerReachability,
 			"ValidateRequiredImagesPresent":          node.ValidateRequiredImagesPresent,
 			"WaitForExistingClusterIfNeeded":         node.WaitForExistingClusterIfNeeded,

clitools/pkg/bootstrap/runner.go

@@ -62,6 +62,16 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Configure default CNI",
 				Desc:   "Install or configure default container networking (CNI bridge, IPAM, etc.)",
 			},
+			{
+				RegKey: "MountAltImageStore",
+				Name:   "Mount alt image store for CRI-O. Needed for upgrade.",
+				Desc:   "Will be unmount after kubeadm upgrade apply",
+			},
+			{
+				RegKey: "EngageControlGate",
+				Name:   "Engage the control gate",
+				Desc:   "Prevents agent polling resources prematurely",
+			},
 			{
 				RegKey: "StartCRIO",
 				Name:   "Start CRI-O runtime",
@@ -127,6 +137,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Run kubeadm upgrade node",
 				Desc:   "Upgrade node components (kubelet, config) to match control plane",
 			},
+			{
+				RegKey: "UnmountAltImageStore",
+				Name:   "Unmount alt image store",
+				Desc:   "Rewrite CRIO storage.conf. Then restart CRIO. Then unmount.",
+			},
 			{
 				RegKey: "ApplyLocalNodeMetadataIfPossible",
 				Name:   "Apply node metadata",
@@ -147,6 +162,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Apply daemonset for control agent",
 				Desc:   "Control agent handles OSUpgrade resources",
 			},
+			{
+				RegKey: "ReleaseControlGate",
+				Name:   "Release the control gate",
+				Desc:   "Allow agent to start polling resources",
+			},
 		},
 	}
 }

clitools/pkg/cmd/agent/agent.go

@@ -3,6 +3,8 @@ package agent
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -50,6 +52,11 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
 				return fmt.Errorf("node name is empty in rendered config")
 			}
 
+			ctx := cmd.Context()
+			if err := waitForControlGate(ctx, envFile, 2*time.Second); err != nil {
+				return fmt.Errorf("wait for control gate to release: %w", err)
+			}
+
 			klog.InfoS("starting agent",
 				"node", cfg.Spec.NodeName,
 				"namespace", namespace,
@@ -62,7 +69,6 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
 				return fmt.Errorf("create kube clients: %w", err)
 			}
 
-			ctx := cmd.Context()
 			return runPollLoop(ctx, clients, namespace, cfg.Spec.NodeName, pollInterval)
 		},
 	}
@@ -74,6 +80,36 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
 	return cmd
 }
 
+func waitForControlGate(ctx context.Context, envFile string, pollInterval time.Duration) error {
+	dir := filepath.Dir(envFile)
+	marker := filepath.Join(dir, ".control-gate")
+
+	if pollInterval <= 0 {
+		pollInterval = 2 * time.Second
+	}
+
+	ticker := time.NewTicker(pollInterval)
+	defer ticker.Stop()
+
+	for {
+		_, err := os.Stat(marker)
+		if err == nil {
+			klog.InfoS("Control gate is present; waiting before starting poll loop", "path", marker)
+		} else if os.IsNotExist(err) {
+			klog.InfoS("Control gate not present; starting poll loop", "path", marker)
+			return nil
+		} else {
+			return fmt.Errorf("stat upgrade marker %s: %w", marker, err)
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
 func runPollLoop(ctx context.Context, clients *kube.Clients, namespace, nodeName string, interval time.Duration) error {
 	gvr := schema.GroupVersionResource{
 		Group:    monov1alpha1.Group,

clitools/pkg/controller/osimage/helpers.go

@@ -1,5 +1,20 @@
 package osimage
 
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+)
+
+var (
+	bootStateOnce sync.Once
+	bootState     map[string]string
+	bootStateErr  error
+)
+
 func PercentOf(done, total int64) int64 {
 	if total <= 0 {
 		return 0
@@ -13,3 +28,34 @@ func PercentOf(done, total int64) int64 {
 	}
 	return p
 }
+
+func ReadBootState() (map[string]string, error) {
+	bootStateOnce.Do(func() {
+		data, err := os.ReadFile(monov1alpha1.BootStateFile)
+		if err != nil {
+			bootStateErr = err
+			return
+		}
+
+		out := make(map[string]string)
+
+		for _, line := range strings.Split(string(data), "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" || strings.HasPrefix(line, "#") {
+				continue
+			}
+
+			k, v, ok := strings.Cut(line, "=")
+			if !ok {
+				bootStateErr = fmt.Errorf("invalid line: %q", line)
+				return
+			}
+
+			out[strings.TrimSpace(k)] = strings.TrimSpace(v)
+		}
+
+		bootState = out
+	})
+
+	return bootState, bootStateErr
+}

clitools/pkg/controller/osupgrade/handler.go

@@ -111,7 +111,7 @@ func handleOSUpgradeLocked(ctx context.Context, clients *kube.Clients,
 		"resolvedTarget", plan.ResolvedTarget,
 		"steps", len(plan.Path),
 		"currentVersion", buildinfo.KubeVersion,
-		"fSHA256irstVersion", first.Version,
+		"firstVersion", first.Version,
 		"firstURL", first.URL,
 		"size", first.Size,
 	)
@@ -126,7 +126,7 @@ func handleOSUpgradeLocked(ctx context.Context, clients *kube.Clients,
 
 	imageOptions := osimage.ApplyOptions{
 		URL:               first.URL,
-		TargetPath:        "/dev/mksaltpart",
+		TargetPath:        monov1alpha1.AltPartDeviceLink,
 		ExpectedRawSHA256: imageSHA,
 		ExpectedRawSize:   first.Size,
 		BufferSize:        6 * 1024 * 1024,

clitools/pkg/node/crio.go

@@ -5,9 +5,14 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
-	"k8s.io/klog/v2"
 	system "example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
+)
+
+const (
+	storageConfPath = "/etc/containers/storage.conf"
 )
 
 func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
@@ -60,3 +65,56 @@ func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
 func StartCRIO(ctx context.Context, n *NodeContext) error {
 	return system.EnsureServiceRunning(ctx, n.SystemRunner, "crio")
 }
+
+func RestartCRIO(ctx context.Context, nctx *NodeContext) error {
+	_, err := nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"rc-service",
+		[]string{"crio", "restart"},
+		system.RunOptions{
+			Timeout:      60 * time.Second,
+			OnStdoutLine: func(line string) { klog.Infof("[crio] %s", line) },
+			OnStderrLine: func(line string) { klog.Infof("[crio] %s", line) },
+		},
+	)
+	return err
+}
+
+func writeCRIOStorageConfig(ctx context.Context, nctx *NodeContext, altSource string) error {
+
+	additionalStores := []string{
+		"/usr/lib/monok8s/imagestore",
+	}
+
+	if altSource != "" {
+		additionalStores = append(additionalStores, altSource)
+	}
+
+	var b strings.Builder
+	b.WriteString("# Generated File. DO NOT MODIFY.\n")
+	b.WriteString("[storage]\n")
+	b.WriteString("driver = \"overlay\"\n")
+	b.WriteString("runroot = \"/run/containers/storage\"\n")
+	b.WriteString("graphroot = \"/var/lib/containers/storage\"\n\n")
+	b.WriteString("[storage.options]\n")
+	b.WriteString("additionalimagestores = [\n")
+	for _, s := range additionalStores {
+		b.WriteString(fmt.Sprintf("  %q,\n", s))
+	}
+	b.WriteString("]\n")
+
+	content := b.String()
+
+	path := storageConfPath
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
+		return fmt.Errorf("write temp storage.conf: %w", err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("replace storage.conf: %w", err)
+	}
+
+	klog.InfoS("wrote CRI-O storage config", "path", path, "additionalImageStores", additionalStores)
+	return nil
+}

clitools/pkg/node/fs.go

@@ -0,0 +1,32 @@
+package node
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+)
+
+func EngageControlGate(ctx context.Context, nctx *NodeContext) error {
+	gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+	f, err := os.OpenFile(gateFile, os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("touch %s: %w", gateFile, err)
+	}
+	defer f.Close()
+
+	return nil
+}
+
+func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error {
+	gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+	if err := os.Remove(gateFile); err != nil {
+		return fmt.Errorf("relate control gate: %w", err)
+	}
+
+	return nil
+}

clitools/pkg/node/kubeadm.go

@@ -732,11 +732,6 @@ func RunKubeadmJoin(ctx context.Context, nctx *NodeContext) error {
 	return nil
 }
 
-func RunKubeadmUpgradeApply(context.Context, *NodeContext) error {
-	klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply")
-	return nil
-}
-
 func RunKubeadmUpgradeNode(context.Context, *NodeContext) error {
 	klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node")
 	return nil

clitools/pkg/node/kubeadm_upgrade.go

@@ -0,0 +1,49 @@
+package node
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	system "example.com/monok8s/pkg/system"
+)
+
+func RunKubeadmUpgradeApply(ctx context.Context, nctx *NodeContext) error {
+
+	if nctx.BootstrapState == nil {
+		return errors.New("BootstrapState is nil. Please run earlier steps first")
+	}
+
+	if nctx.BootstrapState.Action != BootstrapActionUpgradeControlPlane {
+		klog.V(4).Infof("skipped for %s", nctx.BootstrapState.Action)
+		return nil
+	}
+
+	if strings.TrimSpace(tmpKubeadmInitConf) == "" {
+		return fmt.Errorf("tmp kubeadm config path is empty")
+	}
+
+	_, err := nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"kubeadm",
+		[]string{"upgrade", "apply", "-y", nctx.Config.Spec.KubernetesVersion},
+		system.RunOptions{
+			Timeout: 15 * time.Minute,
+			OnStdoutLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+			OnStderrLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("run kubeadm upgrade apply: %w", err)
+	}
+
+	return nil
+}

clitools/pkg/node/mounts.go

@@ -0,0 +1,194 @@
+package node
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+	system "example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
+)
+
+const (
+	altRootMount     = "/run/altrootfs"
+	altImageStoreSrc = "/run/altrootfs/usr/lib/monok8s/imagestore"
+)
+
+func MountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+	// Make mountpoints first.
+	if err := os.MkdirAll(altRootMount, 0o755); err != nil {
+		return fmt.Errorf("mkdir %s: %w", altRootMount, err)
+	}
+
+	altDev := monov1alpha1.AltPartDeviceLink
+	if mounted, err := isMounted(altRootMount); err != nil {
+		return fmt.Errorf("check mount %s: %w", altRootMount, err)
+	} else if !mounted {
+		klog.InfoS("mounting alt rootfs", "device", altDev, "target", altRootMount)
+		if _, err := nctx.SystemRunner.RunWithOptions(
+			ctx,
+			"mount",
+			[]string{"-o", "ro", altDev, altRootMount},
+			system.RunOptions{Timeout: 30 * time.Second},
+		); err != nil {
+			return fmt.Errorf("mount alt rootfs %s on %s: %w", altDev, altRootMount, err)
+		}
+	}
+
+	// If the alt imagestore doesn't exist, don't fail hard unless you want strict behavior.
+	st, err := os.Stat(altImageStoreSrc)
+	if err != nil {
+
+		// Unmount immediately
+		_ = safeUnmount(ctx, nctx, altRootMount)
+
+		if os.IsNotExist(err) {
+			klog.InfoS(
+				"alt imagestore not found; proceeding without it",
+				"path", altImageStoreSrc,
+			)
+			err = writeCRIOStorageConfig(ctx, nctx, "")
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+
+		return fmt.Errorf("stat alt imagestore %s: %w", altImageStoreSrc, err)
+	}
+
+	if !st.IsDir() {
+		return fmt.Errorf("alt imagestore exists but is not a directory: %s", altImageStoreSrc)
+	}
+
+	err = writeCRIOStorageConfig(ctx, nctx, altImageStoreSrc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func UnmountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+
+	mounted, err := isMounted(altRootMount)
+	if err != nil {
+		return err
+	}
+
+	if mounted {
+		if err := writeCRIOStorageConfig(ctx, nctx, ""); err != nil {
+			return err
+		}
+		err := RestartCRIO(ctx, nctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	var errs []string
+
+	if err := safeUnmount(ctx, nctx, altRootMount); err != nil {
+		errs = append(errs, err.Error())
+	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf(strings.Join(errs, "; "))
+	}
+	return nil
+}
+
+func isMounted(path string) (bool, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return false, fmt.Errorf("open /proc/self/mountinfo: %w", err)
+	}
+	defer f.Close()
+
+	target := filepath.Clean(path)
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		fields := strings.Fields(line)
+		if len(fields) < 5 {
+			continue
+		}
+
+		mountPoint, err := unescapeMountInfo(fields[4])
+		if err != nil {
+			return false, fmt.Errorf("decode mountpoint %q: %w", fields[4], err)
+		}
+
+		if filepath.Clean(mountPoint) == target {
+			return true, nil
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return false, fmt.Errorf("scan /proc/self/mountinfo: %w", err)
+	}
+
+	return false, nil
+}
+
+func safeUnmount(ctx context.Context, nctx *NodeContext, path string) error {
+	mounted, err := isMounted(path)
+	if err != nil {
+		return fmt.Errorf("check mountpoint %s: %w", path, err)
+	}
+	if !mounted {
+		return nil
+	}
+
+	klog.InfoS("unmounting", "target", path)
+	_, err = nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"umount",
+		[]string{path},
+		system.RunOptions{Timeout: 30 * time.Second},
+	)
+	if err != nil {
+		return fmt.Errorf("umount %s: %w", path, err)
+	}
+	return nil
+}
+
+func unescapeMountInfo(s string) (string, error) {
+	var b strings.Builder
+	b.Grow(len(s))
+
+	for i := 0; i < len(s); i++ {
+		if s[i] != '\\' {
+			b.WriteByte(s[i])
+			continue
+		}
+
+		if i+3 >= len(s) {
+			return "", fmt.Errorf("truncated escape sequence")
+		}
+
+		esc := s[i+1 : i+4]
+		switch esc {
+		case "040":
+			b.WriteByte(' ')
+		case "011":
+			b.WriteByte('\t')
+		case "012":
+			b.WriteByte('\n')
+		case "134":
+			b.WriteByte('\\')
+		default:
+			return "", fmt.Errorf("unsupported escape sequence \\%s", esc)
+		}
+
+		i += 3
+	}
+
+	return b.String(), nil
+}

clitools/pkg/node/uboot/ab.go

@@ -2,9 +2,14 @@ package uboot
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+	"example.com/monok8s/pkg/controller/osimage"
 	"example.com/monok8s/pkg/node"
 )
 
@@ -15,15 +20,69 @@ func ConfigureABBoot(ctx context.Context, nctx *node.NodeContext) error {
 		return fmt.Errorf("get current executable path: %w", err)
 	}
 
+	doWrite := false
 	writer := NewFWEnvWriter(HostFWEnvCfgPath, exePath)
 
-	// TODO: configurable from cluster.env
+	bootPart, err := writer.GetEnv(ctx, "boot_part")
+	if err != nil {
+		if errors.Is(err, ErrEnvNotFound) {
+
+			state, err := osimage.ReadBootState()
+			if err != nil {
+				return fmt.Errorf("read boot state: %w", err)
+			}
+
+			bootPart := state["BOOT_PART"]
+			if bootPart == "" {
+				return fmt.Errorf("BOOT_PART missing")
+			}
+
+			doWrite = true
+		} else {
+			return fmt.Errorf("get boot_part: %w", err)
+		}
+	}
+
+	bootSource, err := writer.GetEnv(ctx, "boot_source")
+	if err != nil {
+		if errors.Is(err, ErrEnvNotFound) {
+			target, err := os.Readlink(monov1alpha1.AltPartDeviceLink)
+			if err != nil {
+				return fmt.Errorf("readlink %q: %w", monov1alpha1.AltPartDeviceLink, err)
+			}
+
+			if !filepath.IsAbs(target) {
+				target = filepath.Join(filepath.Dir(monov1alpha1.AltPartDeviceLink), target)
+			}
+
+			resolved, err := filepath.EvalSymlinks(target)
+			if err != nil {
+				return fmt.Errorf("resolve %q: %w", target, err)
+			}
+
+			base := filepath.Base(resolved)
+
+			if strings.HasPrefix(base, "mmcblk") {
+				bootSource = "emmc"
+			} else {
+				bootSource = "usb"
+			}
+			doWrite = true
+		} else {
+			return fmt.Errorf("get boot_source: %w", err)
+		}
+	}
+
+	if doWrite {
 		return writer.EnsureBootEnv(ctx, BootEnvConfig{
-		BootSource: "usb",
-		BootPart:   "A",
+			BootSource: bootSource,
+			BootPart:   bootPart,
 		})
 	}
 
+	return nil
+}
+
 // This is called by the agent controller/osupgrade/handler.go
 func ConfigureNextBoot(ctx context.Context, fwEnvCfgPath string) error {
 

clitools/pkg/node/uboot/env_writer.go

@@ -2,6 +2,7 @@ package uboot
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -11,6 +12,8 @@ import (
 	"example.com/monok8s/pkg/system"
 )
 
+var ErrEnvNotFound = errors.New("fw env not found")
+
 func NewFWEnvWriter(configPath string, ctlPath string) *FWEnvWriter {
 	return &FWEnvWriter{
 		Runner: system.NewRunner(system.RunnerConfig{
@@ -38,9 +41,13 @@ func (w *FWEnvWriter) GetEnv(ctx context.Context, key string) (string, error) {
 	res, err := w.Runner.RunWithOptions(ctx, w.CtlPath, args, system.RunOptions{Quiet: true})
 	if err != nil {
 		if res != nil {
+			if strings.Contains(res.Stderr, "not defined") {
+				return "", ErrEnvNotFound
+			} else {
 				return "", fmt.Errorf("fw-printenv %q: %w (stdout=%q stderr=%q)",
 					key, err, strings.TrimSpace(res.Stdout), strings.TrimSpace(res.Stderr))
 			}
+		}
 		return "", fmt.Errorf("fw-printenv %q: %w", key, err)
 	}
 

docs/flashing-usb.md

@@ -23,8 +23,24 @@ bootm 0x80000000
 flash-emmc.sh
 ```
 
-4. If it boots, create the A/B deployment scheme
- - (WORK IN PROGRESS)
+6. Reboot into uboot, boot using the following commands
+```
+setenv boot_source emmc
+setenv boot_part A
+
+setenv rootpart 2;
+
+setenv bootdev 0:${rootpart};
+setenv rootdev emmc:2;
+
+setenv kernel_addr_r 0xa0000000;
+
+setenv bootargs "${bootargs_console} root=${rootdev} bootpart=${boot_part} rw rootwait rootfstype=ext4";
+ext4load mmc ${bootdev} ${kernel_addr_r} /boot/kernel.itb && bootm ${kernel_addr_r};'
+```
+
+7. tail /var/log/monok8s/bootstrap.log
+
 
 ## Flashing into USB
 On MacOS

initramfs/rootfs-extra/init

@@ -221,7 +221,7 @@ log "Booting kernel took $(cut -d' ' -f1 /proc/uptime) seconds."
 
 . /etc/build-info || panic "failed to source /etc/build-info"
 
-wait_for_partnames 30 rootfsA rootfsB data || panic "failed to wait for fs"
+wait_for_partnames 5 rootfsA rootfsB data || panic "failed to wait for fs"
 
 ROOT_CMD="$(get_cmdline_arg root || true)"
 BOOT_PART="$(get_cmdline_arg bootpart || true)"
@@ -283,6 +283,15 @@ if [ -n "$ALT_PART" ]; then
     ln -sf "$ALT_PART" /dev/mksaltpart
 fi
 
+mkdir -p /run/monok8s
+cat > /run/monok8s/boot-state.env <<EOF
+BOOT_PART=$BOOT_PART
+ROOT_DEV=$ROOT_DEV
+DATA_DEV=$DATA_DEV
+EOF
+
+mkdir -p /run/altrootfs
+
 mount_or_panic --move /dev /newroot/dev
 mount_or_panic --move /proc /newroot/proc
 mount_or_panic --move /sys /newroot/sys