diff --git a/README.md b/README.md index 9c643a8..4c736c5 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,11 @@ https://docs.mono.si/gateway-development-kit/getting-started ### Kubernetes * OSUpgrade * [x] Control Plane - kubeadm upgrade apply - * [ ] Worker node - kubeadm upgrade node * Upgrade chain * [x] 1.33.3 -> 1.33.10 - * [ ] 1.33.10 -> 1.34.1 - * [ ] 1.34.1 -> 1.34.6 - * [ ] 1.34.6 -> 1.35.1 - * [ ] 1.35.1 -> 1.35.3 + * [x] 1.33.10 -> 1.34.6 + * [x] 1.34.6 -> 1.35.3 + * [ ] Worker node - kubeadm upgrade node * CNI * [x] default bridge-cni * [ ] Cilium diff --git a/alpine/build-rootfs.sh b/alpine/build-rootfs.sh index 2978a99..f9d4a3f 100755 --- a/alpine/build-rootfs.sh +++ b/alpine/build-rootfs.sh @@ -17,6 +17,7 @@ mkdir -p \ "$ROOTFS/build" \ "$ROOTFS/var/cache/apk" \ "$ROOTFS/usr/lib/monok8s/crds" \ + "$ROOTFS/usr/lib/monok8s/migrations.d/k8s" \ "$ROOTFS/opt/monok8s/config" mount --bind /var/cache/apk "$ROOTFS/var/cache/apk" @@ -30,6 +31,15 @@ cp /etc/resolv.conf "$ROOTFS/etc/resolv.conf" cp /build/crio.tar.gz "$ROOTFS/build/" cp /build/crds/*.yaml "$ROOTFS/usr/lib/monok8s/crds" +KUBE_MINOR=$(printf '%s\n' "$KUBE_VERSION" | sed -E 's/^v?([0-9]+\.[0-9]+).*/\1/') +MIG_SRC="/build/migrations/k8s/$KUBE_MINOR" +MIG_DST="$ROOTFS/usr/lib/monok8s/migrations.d/k8s/$KUBE_MINOR" + +if [ -d "$MIG_SRC" ]; then + mkdir -p "$MIG_DST" + cp -a "$MIG_SRC"/. "$MIG_DST"/ +fi + chroot "$ROOTFS" /bin/sh -c "ln -s /var/cache/apk /etc/apk/cache" # chroot "$ROOTFS" /bin/sh -c "apk update" chroot "$ROOTFS" /bin/sh -c "apk add bash curl" diff --git a/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh b/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh new file mode 100755 index 0000000..c4d73c2 --- /dev/null +++ b/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +# Kubernetes removed the kubelet flag: +# --pod-infra-container-image +# +# Timeline: +# - Deprecated before v1.27 +# - Removed in newer kubelet versions (>=1.27+) +# - kubeadm may still write it into: +# /var/lib/kubelet/kubeadm-flags.env +# +# This causes kubelet to fail with: +# "unknown flag: --pod-infra-container-image" +# +# References: +# - https://github.com/kubernetes/kubeadm/issues/3281 +# - https://github.com/kubernetes/kubernetes/pull/122739 +# - https://kubernetes.io/blog/2022/04/07/upcoming-changes-in-kubernetes-1-24/ +# +# Root cause: +# - Sandbox (pause) image is now managed by CRI (containerd/CRI-O), +# not kubelet flags. +# +# Fix: +# - Strip the flag from kubeadm-flags.env during upgrade + +FILE=/var/lib/kubelet/kubeadm-flags.env + +[ -f "$FILE" ] || exit 0 +grep -q -- '--pod-infra-container-image=' "$FILE" || exit 0 + +sed -i 's/ --pod-infra-container-image=[^"]*//g' "$FILE" + +echo "Removed deprecated kubelet flag --pod-infra-container-image from $FILE" diff --git a/alpine/rootfs-extra/etc/local.d/monok8s.start b/alpine/rootfs-extra/etc/local.d/monok8s.start index 2f88bd9..4e76376 100755 --- a/alpine/rootfs-extra/etc/local.d/monok8s.start +++ b/alpine/rootfs-extra/etc/local.d/monok8s.start @@ -5,22 +5,37 @@ exec >>/var/log/monok8s/boot.log 2>&1 export PATH="/usr/local/bin:/usr/local/sbin:$PATH" +MIGRATIONS_LIB=/usr/lib/monok8s/lib/migrations.sh +CONFIG_DIR=/opt/monok8s/config +BOOT_STATE=/run/monok8s/boot-state.env +BOOTPART_FILE="$CONFIG_DIR/.bootpart" +MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state" + mkdir -p /dev/hugepages mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages -echo 640 > /proc/sys/vm/nr_hugepages +echo 256 > /proc/sys/vm/nr_hugepages -CUR=$(grep '^BOOT_PART=' /run/monok8s/boot-state.env | cut -d= -f2-) -LAST=$(cat /opt/monok8s/config/.bootpart 2>/dev/null || true) +CUR=$(grep '^BOOT_PART=' "$BOOT_STATE" | cut -d= -f2-) +LAST=$(cat "$BOOTPART_FILE" 2>/dev/null || true) +slot_changed=0 if [ "$CUR" != "$LAST" ]; then - echo "Slot changed ($LAST -> $CUR), cleaning runtime state" - - rm -rf /var/lib/containers \ - /var/lib/kubelet/pods \ - /var/lib/kubelet/plugins \ - /var/lib/kubelet/device-plugins - - mkdir -p /var/lib/containers /var/lib/kubelet + slot_changed=1 + echo "Slot changed ($LAST -> $CUR)" fi -/usr/local/bin/ctl init --env-file /opt/monok8s/config/cluster.env >>/var/log/monok8s/bootstrap.log 2>&1 & +# shellcheck source=/dev/null +. "$MIGRATIONS_LIB" + +if [ "$slot_changed" -eq 1 ]; then + monok8s_cleanup_runtime_state +fi + +K8S_MINOR="$(monok8s_detect_k8s_minor || true)" +if [ -n "$K8S_MINOR" ]; then + monok8s_run_migration_dir \ + "/usr/lib/monok8s/migrations.d/k8s/$K8S_MINOR" \ + "$MIGRATION_STATE_DIR/k8s/$K8S_MINOR" +fi + +/usr/local/bin/ctl init --env-file "$CONFIG_DIR/cluster.env" >>/var/log/monok8s/bootstrap.log 2>&1 & diff --git a/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh b/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh new file mode 100755 index 0000000..2758439 --- /dev/null +++ b/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +monok8s_detect_k8s_minor() { + local env_file=${1:-} + local version major_minor + + version=$(/usr/local/bin/ctl version -k 2>/dev/null || true) + + [ -n "$version" ] || return 1 + + version=${version#v} + major_minor=$(printf '%s\n' "$version" | cut -d. -f1,2) + + [ -n "$major_minor" ] || return 1 + printf '%s\n' "$major_minor" +} + +monok8s_cleanup_runtime_state() { + echo "Cleaning runtime state" + + rm -rf \ + /var/lib/containers \ + /var/lib/cni \ + /var/lib/kubelet/pods \ + /var/lib/kubelet/plugins \ + /var/lib/kubelet/plugins_registry \ + /var/lib/kubelet/device-plugins \ + /run/containers \ + /run/netns + + mkdir -p \ + /var/lib/containers \ + /var/lib/kubelet \ + /var/lib/cni +} + +monok8s_run_migration_dir() { + dir=$1 + state_dir=$2 + + [ -d "$dir" ] || return 0 + mkdir -p "$state_dir" + + for script in "$dir"/*.sh; do + [ -e "$script" ] || continue + + name=$(basename "$script") + stamp="$state_dir/$name.done" + + if [ -e "$stamp" ]; then + continue + fi + + echo "Running migration: $script" + sh "$script" + : > "$stamp" + done +} diff --git a/clitools/pkg/cmd/version/version.go b/clitools/pkg/cmd/version/version.go index 82406c5..f5ef2ec 100644 --- a/clitools/pkg/cmd/version/version.go +++ b/clitools/pkg/cmd/version/version.go @@ -1,22 +1,76 @@ -package apply +package version import ( + "encoding/json" "fmt" "github.com/spf13/cobra" - buildInfo "example.com/monok8s/pkg/buildinfo" + buildinfo "example.com/monok8s/pkg/buildinfo" ) +type versionInfo struct { + Version string `json:"version"` + GitRevision string `json:"gitRevision"` + Timestamp string `json:"timestamp"` + KubeVersion string `json:"kubernetesVersion"` +} + func NewCmdVersion() *cobra.Command { + var ( + shortOutput bool + jsonOutput bool + kubernetesOutput bool + ) + cmd := &cobra.Command{ Use: "version", - Short: "Print the version information", + Short: "Print version information", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + info := versionInfo{ + Version: buildinfo.Version, + GitRevision: buildinfo.GitRevision, + Timestamp: buildinfo.Timestamp, + KubeVersion: buildinfo.KubeVersion, + } - _, err := fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s %s(%s)", buildInfo.Version, buildInfo.GitRevision, buildInfo.Timestamp)) - return err + out := cmd.OutOrStdout() + + switch { + case jsonOutput: + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(info) + + case kubernetesOutput: + _, err := fmt.Fprintln(out, info.KubeVersion) + return err + + case shortOutput: + _, err := fmt.Fprintln(out, info.Version) + return err + + default: + _, err := fmt.Fprintf( + out, + "Version: %s\nGit commit: %s\nBuilt at: %s\nKubernetes: %s\n", + info.Version, + info.GitRevision, + info.Timestamp, + info.KubeVersion, + ) + return err + } }, } + + flags := cmd.Flags() + flags.BoolVar(&shortOutput, "short", false, "Show only the application version") + flags.BoolVar(&jsonOutput, "json", false, "Show version information as JSON") + flags.BoolVarP(&kubernetesOutput, "kubernetes", "k", false, "Show only the Kubernetes version this binary was built for") + + cmd.MarkFlagsMutuallyExclusive("short", "json", "kubernetes") + return cmd } diff --git a/clitools/pkg/controller/osimage/write_throttle_other.go b/clitools/pkg/controller/osimage/write_throttle_other.go index 3c3feea..57dd3df 100644 --- a/clitools/pkg/controller/osimage/write_throttle_other.go +++ b/clitools/pkg/controller/osimage/write_throttle_other.go @@ -15,4 +15,5 @@ func newNoopAdaptiveWriteController() *adaptiveWriteController { } func (c *adaptiveWriteController) Wait(ctx context.Context, n int) error { return nil } -func (c *adaptiveWriteController) ObserveWrite(n int, dur interface{}) {} +func (c *adaptiveWriteController) ObserveWrite(n int) {} +func (c *adaptiveWriteController) ObserveSync() {} diff --git a/clitools/pkg/controller/osupgrade/planner.go b/clitools/pkg/controller/osupgrade/planner.go index 29b52a4..7111fd2 100644 --- a/clitools/pkg/controller/osupgrade/planner.go +++ b/clitools/pkg/controller/osupgrade/planner.go @@ -224,9 +224,9 @@ func calculatePath(current, target string, available []string) ([]string, error) add(latestCurMinor) } - // Step 2: walk each intermediate minor using the lowest available patch in that minor. + // Step 2: walk each intermediate minor using the latest available patch in that minor. for minor := cur.Minor + 1; minor < tgt.Minor; minor++ { - bridge, ok := lowestPatchInMinor(versions, cur.Major, minor) + bridge, ok := latestAnyPatchInMinor(versions, cur.Major, minor) if !ok { return nil, fmt.Errorf("no available bridge version for v%d.%d.x", cur.Major, minor) } @@ -239,6 +239,23 @@ func calculatePath(current, target string, available []string) ([]string, error) return versionsToStrings(path), nil } +func latestAnyPatchInMinor(versions []Version, major, minor int) (Version, bool) { + var found Version + ok := false + + for _, v := range versions { + if v.Major != major || v.Minor != minor { + continue + } + if !ok || found.Compare(v) < 0 { + found = v + ok = true + } + } + + return found, ok +} + func parseAndSortVersions(raw []string) ([]Version, error) { out := make([]Version, 0, len(raw)) seen := map[string]struct{}{} diff --git a/clitools/pkg/controller/osupgrade/planner_test.go b/clitools/pkg/controller/osupgrade/planner_test.go new file mode 100644 index 0000000..b6b668d --- /dev/null +++ b/clitools/pkg/controller/osupgrade/planner_test.go @@ -0,0 +1,149 @@ +package osupgrade + +import ( + "reflect" + "testing" +) + +func TestCalculatePath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + current string + target string + available []string + want []string + wantErr bool + }{ + { + name: "same version returns nil path", + current: "v1.34.6", + target: "v1.34.6", + available: []string{"v1.34.6"}, + want: nil, + wantErr: false, + }, + { + name: "same minor jumps directly to target", + current: "v1.34.1", + target: "v1.34.6", + available: []string{"v1.34.1", "v1.34.3", "v1.34.6"}, + want: []string{"v1.34.6"}, + wantErr: false, + }, + { + name: "next minor direct jump when no current minor patch available", + current: "v1.34.6", + target: "v1.35.3", + available: []string{"v1.34.6", "v1.35.1", "v1.35.3"}, + want: []string{"v1.35.3"}, + wantErr: false, + }, + { + name: "finish current minor then target", + current: "v1.34.1", + target: "v1.35.3", + available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"}, + want: []string{"v1.34.6", "v1.35.3"}, + wantErr: false, + }, + { + name: "multi minor path uses latest bridge patch", + current: "v1.33.10", + target: "v1.35.3", + available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"}, + want: []string{"v1.34.6", "v1.35.3"}, + wantErr: false, + }, + { + name: "multi minor path finishes current minor and latest bridge patch", + current: "v1.33.1", + target: "v1.35.3", + available: []string{"v1.33.5", "v1.33.9", "v1.34.1", "v1.34.6", "v1.35.3"}, + want: []string{"v1.33.9", "v1.34.6", "v1.35.3"}, + wantErr: false, + }, + { + name: "duplicates in available are ignored", + current: "v1.33.10", + target: "v1.35.3", + available: []string{"v1.34.6", "v1.34.6", "v1.35.3", "v1.35.3"}, + want: []string{"v1.34.6", "v1.35.3"}, + wantErr: false, + }, + { + name: "target missing returns error", + current: "v1.34.6", + target: "v1.35.3", + available: []string{"v1.34.6", "v1.35.1"}, + wantErr: true, + }, + { + name: "missing bridge minor returns error", + current: "v1.33.10", + target: "v1.35.3", + available: []string{"v1.35.3"}, + wantErr: true, + }, + { + name: "downgrade not supported", + current: "v1.35.3", + target: "v1.34.6", + available: []string{"v1.34.6", "v1.35.3"}, + wantErr: true, + }, + { + name: "cross major not supported", + current: "v1.35.3", + target: "v2.0.0", + available: []string{"v1.35.3", "v2.0.0"}, + wantErr: true, + }, + { + name: "invalid current version returns error", + current: "garbage", + target: "v1.35.3", + available: []string{"v1.35.3"}, + wantErr: true, + }, + { + name: "invalid target version returns error", + current: "v1.34.6", + target: "wat", + available: []string{"v1.34.6", "v1.35.3"}, + wantErr: true, + }, + { + name: "invalid available version returns error", + current: "v1.34.6", + target: "v1.35.3", + available: []string{"v1.34.6", "broken", "v1.35.3"}, + wantErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := calculatePath(tt.current, tt.target, tt.available) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil; path=%v", got) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("calculatePath(%q, %q, %v)\n got: %v\n want: %v", + tt.current, tt.target, tt.available, got, tt.want) + } + }) + } +} diff --git a/clitools/pkg/node/fs.go b/clitools/pkg/node/fs.go index 8eab227..79ff8b1 100644 --- a/clitools/pkg/node/fs.go +++ b/clitools/pkg/node/fs.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" + "k8s.io/klog/v2" + monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" "example.com/monok8s/pkg/controller/osimage" ) @@ -26,7 +28,7 @@ func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error { gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate") if err := os.Remove(gateFile); err != nil { - return fmt.Errorf("relate control gate: %w", err) + return fmt.Errorf("release control gate: %w", err) } return WriteLastState(ctx, nctx) @@ -46,6 +48,8 @@ func WriteLastState(ctx context.Context, nctx *NodeContext) error { return fmt.Errorf("BOOT_PART missing") } + klog.Infof("Writing last state: %+v", bootPart) + tmp := stBootPart + ".tmp" if err := os.WriteFile(tmp, []byte(bootPart+"\n"), 0o644); err != nil { return err diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile index fbee291..6378bd9 100644 --- a/docker/alpine.Dockerfile +++ b/docker/alpine.Dockerfile @@ -31,6 +31,7 @@ COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubec # COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/ # COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/ COPY alpine/rootfs-extra ./rootfs-extra +COPY alpine/migrations ./migrations COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh COPY alpine/*.sh / RUN chmod +x /out/rootfs/usr/local/bin/*