Initial upgrade flow for control-plane

clitools/pkg/node/crio.go

@@ -5,9 +5,14 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
-	"k8s.io/klog/v2"
 	system "example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
+)
+
+const (
+	storageConfPath = "/etc/containers/storage.conf"
 )
 
 func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
@@ -60,3 +65,56 @@ func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
 func StartCRIO(ctx context.Context, n *NodeContext) error {
 	return system.EnsureServiceRunning(ctx, n.SystemRunner, "crio")
 }
+
+func RestartCRIO(ctx context.Context, nctx *NodeContext) error {
+	_, err := nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"rc-service",
+		[]string{"crio", "restart"},
+		system.RunOptions{
+			Timeout:      60 * time.Second,
+			OnStdoutLine: func(line string) { klog.Infof("[crio] %s", line) },
+			OnStderrLine: func(line string) { klog.Infof("[crio] %s", line) },
+		},
+	)
+	return err
+}
+
+func writeCRIOStorageConfig(ctx context.Context, nctx *NodeContext, altSource string) error {
+
+	additionalStores := []string{
+		"/usr/lib/monok8s/imagestore",
+	}
+
+	if altSource != "" {
+		additionalStores = append(additionalStores, altSource)
+	}
+
+	var b strings.Builder
+	b.WriteString("# Generated File. DO NOT MODIFY.\n")
+	b.WriteString("[storage]\n")
+	b.WriteString("driver = \"overlay\"\n")
+	b.WriteString("runroot = \"/run/containers/storage\"\n")
+	b.WriteString("graphroot = \"/var/lib/containers/storage\"\n\n")
+	b.WriteString("[storage.options]\n")
+	b.WriteString("additionalimagestores = [\n")
+	for _, s := range additionalStores {
+		b.WriteString(fmt.Sprintf("  %q,\n", s))
+	}
+	b.WriteString("]\n")
+
+	content := b.String()
+
+	path := storageConfPath
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
+		return fmt.Errorf("write temp storage.conf: %w", err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("replace storage.conf: %w", err)
+	}
+
+	klog.InfoS("wrote CRI-O storage config", "path", path, "additionalImageStores", additionalStores)
+	return nil
+}

clitools/pkg/node/fs.go

@@ -0,0 +1,32 @@
+package node
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+)
+
+func EngageControlGate(ctx context.Context, nctx *NodeContext) error {
+	gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+	f, err := os.OpenFile(gateFile, os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("touch %s: %w", gateFile, err)
+	}
+	defer f.Close()
+
+	return nil
+}
+
+func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error {
+	gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+	if err := os.Remove(gateFile); err != nil {
+		return fmt.Errorf("relate control gate: %w", err)
+	}
+
+	return nil
+}

clitools/pkg/node/kubeadm.go

@@ -732,11 +732,6 @@ func RunKubeadmJoin(ctx context.Context, nctx *NodeContext) error {
 	return nil
 }
 
-func RunKubeadmUpgradeApply(context.Context, *NodeContext) error {
-	klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply")
-	return nil
-}
-
 func RunKubeadmUpgradeNode(context.Context, *NodeContext) error {
 	klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node")
 	return nil

clitools/pkg/node/kubeadm_upgrade.go

@@ -0,0 +1,49 @@
+package node
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	system "example.com/monok8s/pkg/system"
+)
+
+func RunKubeadmUpgradeApply(ctx context.Context, nctx *NodeContext) error {
+
+	if nctx.BootstrapState == nil {
+		return errors.New("BootstrapState is nil. Please run earlier steps first")
+	}
+
+	if nctx.BootstrapState.Action != BootstrapActionUpgradeControlPlane {
+		klog.V(4).Infof("skipped for %s", nctx.BootstrapState.Action)
+		return nil
+	}
+
+	if strings.TrimSpace(tmpKubeadmInitConf) == "" {
+		return fmt.Errorf("tmp kubeadm config path is empty")
+	}
+
+	_, err := nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"kubeadm",
+		[]string{"upgrade", "apply", "-y", nctx.Config.Spec.KubernetesVersion},
+		system.RunOptions{
+			Timeout: 15 * time.Minute,
+			OnStdoutLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+			OnStderrLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("run kubeadm upgrade apply: %w", err)
+	}
+
+	return nil
+}

clitools/pkg/node/mounts.go

@@ -0,0 +1,194 @@
+package node
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+	system "example.com/monok8s/pkg/system"
+	"k8s.io/klog/v2"
+)
+
+const (
+	altRootMount     = "/run/altrootfs"
+	altImageStoreSrc = "/run/altrootfs/usr/lib/monok8s/imagestore"
+)
+
+func MountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+	// Make mountpoints first.
+	if err := os.MkdirAll(altRootMount, 0o755); err != nil {
+		return fmt.Errorf("mkdir %s: %w", altRootMount, err)
+	}
+
+	altDev := monov1alpha1.AltPartDeviceLink
+	if mounted, err := isMounted(altRootMount); err != nil {
+		return fmt.Errorf("check mount %s: %w", altRootMount, err)
+	} else if !mounted {
+		klog.InfoS("mounting alt rootfs", "device", altDev, "target", altRootMount)
+		if _, err := nctx.SystemRunner.RunWithOptions(
+			ctx,
+			"mount",
+			[]string{"-o", "ro", altDev, altRootMount},
+			system.RunOptions{Timeout: 30 * time.Second},
+		); err != nil {
+			return fmt.Errorf("mount alt rootfs %s on %s: %w", altDev, altRootMount, err)
+		}
+	}
+
+	// If the alt imagestore doesn't exist, don't fail hard unless you want strict behavior.
+	st, err := os.Stat(altImageStoreSrc)
+	if err != nil {
+
+		// Unmount immediately
+		_ = safeUnmount(ctx, nctx, altRootMount)
+
+		if os.IsNotExist(err) {
+			klog.InfoS(
+				"alt imagestore not found; proceeding without it",
+				"path", altImageStoreSrc,
+			)
+			err = writeCRIOStorageConfig(ctx, nctx, "")
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+
+		return fmt.Errorf("stat alt imagestore %s: %w", altImageStoreSrc, err)
+	}
+
+	if !st.IsDir() {
+		return fmt.Errorf("alt imagestore exists but is not a directory: %s", altImageStoreSrc)
+	}
+
+	err = writeCRIOStorageConfig(ctx, nctx, altImageStoreSrc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func UnmountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+
+	mounted, err := isMounted(altRootMount)
+	if err != nil {
+		return err
+	}
+
+	if mounted {
+		if err := writeCRIOStorageConfig(ctx, nctx, ""); err != nil {
+			return err
+		}
+		err := RestartCRIO(ctx, nctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	var errs []string
+
+	if err := safeUnmount(ctx, nctx, altRootMount); err != nil {
+		errs = append(errs, err.Error())
+	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf(strings.Join(errs, "; "))
+	}
+	return nil
+}
+
+func isMounted(path string) (bool, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return false, fmt.Errorf("open /proc/self/mountinfo: %w", err)
+	}
+	defer f.Close()
+
+	target := filepath.Clean(path)
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		fields := strings.Fields(line)
+		if len(fields) < 5 {
+			continue
+		}
+
+		mountPoint, err := unescapeMountInfo(fields[4])
+		if err != nil {
+			return false, fmt.Errorf("decode mountpoint %q: %w", fields[4], err)
+		}
+
+		if filepath.Clean(mountPoint) == target {
+			return true, nil
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return false, fmt.Errorf("scan /proc/self/mountinfo: %w", err)
+	}
+
+	return false, nil
+}
+
+func safeUnmount(ctx context.Context, nctx *NodeContext, path string) error {
+	mounted, err := isMounted(path)
+	if err != nil {
+		return fmt.Errorf("check mountpoint %s: %w", path, err)
+	}
+	if !mounted {
+		return nil
+	}
+
+	klog.InfoS("unmounting", "target", path)
+	_, err = nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"umount",
+		[]string{path},
+		system.RunOptions{Timeout: 30 * time.Second},
+	)
+	if err != nil {
+		return fmt.Errorf("umount %s: %w", path, err)
+	}
+	return nil
+}
+
+func unescapeMountInfo(s string) (string, error) {
+	var b strings.Builder
+	b.Grow(len(s))
+
+	for i := 0; i < len(s); i++ {
+		if s[i] != '\\' {
+			b.WriteByte(s[i])
+			continue
+		}
+
+		if i+3 >= len(s) {
+			return "", fmt.Errorf("truncated escape sequence")
+		}
+
+		esc := s[i+1 : i+4]
+		switch esc {
+		case "040":
+			b.WriteByte(' ')
+		case "011":
+			b.WriteByte('\t')
+		case "012":
+			b.WriteByte('\n')
+		case "134":
+			b.WriteByte('\\')
+		default:
+			return "", fmt.Errorf("unsupported escape sequence \\%s", esc)
+		}
+
+		i += 3
+	}
+
+	return b.String(), nil
+}

clitools/pkg/node/uboot/ab.go

@@ -2,9 +2,14 @@ package uboot
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 
+	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+	"example.com/monok8s/pkg/controller/osimage"
 	"example.com/monok8s/pkg/node"
 )
 
@@ -15,13 +20,67 @@ func ConfigureABBoot(ctx context.Context, nctx *node.NodeContext) error {
 		return fmt.Errorf("get current executable path: %w", err)
 	}
 
+	doWrite := false
 	writer := NewFWEnvWriter(HostFWEnvCfgPath, exePath)
 
-	// TODO: configurable from cluster.env
-	return writer.EnsureBootEnv(ctx, BootEnvConfig{
-		BootSource: "usb",
-		BootPart:   "A",
-	})
+	bootPart, err := writer.GetEnv(ctx, "boot_part")
+	if err != nil {
+		if errors.Is(err, ErrEnvNotFound) {
+
+			state, err := osimage.ReadBootState()
+			if err != nil {
+				return fmt.Errorf("read boot state: %w", err)
+			}
+
+			bootPart := state["BOOT_PART"]
+			if bootPart == "" {
+				return fmt.Errorf("BOOT_PART missing")
+			}
+
+			doWrite = true
+		} else {
+			return fmt.Errorf("get boot_part: %w", err)
+		}
+	}
+
+	bootSource, err := writer.GetEnv(ctx, "boot_source")
+	if err != nil {
+		if errors.Is(err, ErrEnvNotFound) {
+			target, err := os.Readlink(monov1alpha1.AltPartDeviceLink)
+			if err != nil {
+				return fmt.Errorf("readlink %q: %w", monov1alpha1.AltPartDeviceLink, err)
+			}
+
+			if !filepath.IsAbs(target) {
+				target = filepath.Join(filepath.Dir(monov1alpha1.AltPartDeviceLink), target)
+			}
+
+			resolved, err := filepath.EvalSymlinks(target)
+			if err != nil {
+				return fmt.Errorf("resolve %q: %w", target, err)
+			}
+
+			base := filepath.Base(resolved)
+
+			if strings.HasPrefix(base, "mmcblk") {
+				bootSource = "emmc"
+			} else {
+				bootSource = "usb"
+			}
+			doWrite = true
+		} else {
+			return fmt.Errorf("get boot_source: %w", err)
+		}
+	}
+
+	if doWrite {
+		return writer.EnsureBootEnv(ctx, BootEnvConfig{
+			BootSource: bootSource,
+			BootPart:   bootPart,
+		})
+	}
+
+	return nil
 }
 
 // This is called by the agent controller/osupgrade/handler.go

clitools/pkg/node/uboot/env_writer.go

@@ -2,6 +2,7 @@ package uboot
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -11,6 +12,8 @@ import (
 	"example.com/monok8s/pkg/system"
 )
 
+var ErrEnvNotFound = errors.New("fw env not found")
+
 func NewFWEnvWriter(configPath string, ctlPath string) *FWEnvWriter {
 	return &FWEnvWriter{
 		Runner: system.NewRunner(system.RunnerConfig{
@@ -38,8 +41,12 @@ func (w *FWEnvWriter) GetEnv(ctx context.Context, key string) (string, error) {
 	res, err := w.Runner.RunWithOptions(ctx, w.CtlPath, args, system.RunOptions{Quiet: true})
 	if err != nil {
 		if res != nil {
-			return "", fmt.Errorf("fw-printenv %q: %w (stdout=%q stderr=%q)",
-				key, err, strings.TrimSpace(res.Stdout), strings.TrimSpace(res.Stderr))
+			if strings.Contains(res.Stderr, "not defined") {
+				return "", ErrEnvNotFound
+			} else {
+				return "", fmt.Errorf("fw-printenv %q: %w (stdout=%q stderr=%q)",
+					key, err, strings.TrimSpace(res.Stdout), strings.TrimSpace(res.Stderr))
+			}
 		}
 		return "", fmt.Errorf("fw-printenv %q: %w", key, err)
 	}