Added migrations.d

alpine/build-rootfs.sh

@@ -17,6 +17,7 @@ mkdir -p \
   "$ROOTFS/build" \
   "$ROOTFS/var/cache/apk" \
   "$ROOTFS/usr/lib/monok8s/crds" \
+  "$ROOTFS/usr/lib/monok8s/migrations.d/k8s" \
   "$ROOTFS/opt/monok8s/config"
 
 mount --bind /var/cache/apk "$ROOTFS/var/cache/apk"
@@ -30,6 +31,15 @@ cp /etc/resolv.conf "$ROOTFS/etc/resolv.conf"
 cp /build/crio.tar.gz "$ROOTFS/build/"
 cp /build/crds/*.yaml "$ROOTFS/usr/lib/monok8s/crds"
 
+KUBE_MINOR=$(printf '%s\n' "$KUBE_VERSION" | sed -E 's/^v?([0-9]+\.[0-9]+).*/\1/')
+MIG_SRC="/build/migrations/k8s/$KUBE_MINOR"
+MIG_DST="$ROOTFS/usr/lib/monok8s/migrations.d/k8s/$KUBE_MINOR"
+
+if [ -d "$MIG_SRC" ]; then
+    mkdir -p "$MIG_DST"
+    cp -a "$MIG_SRC"/. "$MIG_DST"/
+fi
+
 chroot "$ROOTFS" /bin/sh -c "ln -s /var/cache/apk /etc/apk/cache"
 # chroot "$ROOTFS" /bin/sh -c "apk update"
 chroot "$ROOTFS" /bin/sh -c "apk add bash curl"

alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+set -eu
+
+# Kubernetes removed the kubelet flag:
+#   --pod-infra-container-image
+#
+# Timeline:
+# - Deprecated before v1.27
+# - Removed in newer kubelet versions (>=1.27+)
+# - kubeadm may still write it into:
+#     /var/lib/kubelet/kubeadm-flags.env
+#
+# This causes kubelet to fail with:
+#   "unknown flag: --pod-infra-container-image"
+#
+# References:
+# - https://github.com/kubernetes/kubeadm/issues/3281
+# - https://github.com/kubernetes/kubernetes/pull/122739
+# - https://kubernetes.io/blog/2022/04/07/upcoming-changes-in-kubernetes-1-24/
+#
+# Root cause:
+# - Sandbox (pause) image is now managed by CRI (containerd/CRI-O),
+#   not kubelet flags.
+#
+# Fix:
+# - Strip the flag from kubeadm-flags.env during upgrade
+
+FILE=/var/lib/kubelet/kubeadm-flags.env
+
+[ -f "$FILE" ] || exit 0
+grep -q -- '--pod-infra-container-image=' "$FILE" || exit 0
+
+sed -i 's/ --pod-infra-container-image=[^"]*//g' "$FILE"
+
+echo "Removed deprecated kubelet flag --pod-infra-container-image from $FILE"

alpine/rootfs-extra/etc/local.d/monok8s.start

@@ -5,22 +5,37 @@ exec >>/var/log/monok8s/boot.log 2>&1
 
 export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
 
+MIGRATIONS_LIB=/usr/lib/monok8s/lib/migrations.sh
+CONFIG_DIR=/opt/monok8s/config
+BOOT_STATE=/run/monok8s/boot-state.env
+BOOTPART_FILE="$CONFIG_DIR/.bootpart"
+MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state"
+
 mkdir -p /dev/hugepages
 mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages
-echo 640 > /proc/sys/vm/nr_hugepages
+echo 256 > /proc/sys/vm/nr_hugepages
 
-CUR=$(grep '^BOOT_PART=' /run/monok8s/boot-state.env | cut -d= -f2-)
-LAST=$(cat /opt/monok8s/config/.bootpart 2>/dev/null || true)
+CUR=$(grep '^BOOT_PART=' "$BOOT_STATE" | cut -d= -f2-)
+LAST=$(cat "$BOOTPART_FILE" 2>/dev/null || true)
 
+slot_changed=0
 if [ "$CUR" != "$LAST" ]; then
-	echo "Slot changed ($LAST -> $CUR), cleaning runtime state"
-
-	rm -rf /var/lib/containers \
-		/var/lib/kubelet/pods \
-		/var/lib/kubelet/plugins \
-		/var/lib/kubelet/device-plugins
-
-	mkdir -p /var/lib/containers /var/lib/kubelet
+	slot_changed=1
+	echo "Slot changed ($LAST -> $CUR)"
 fi
 
-/usr/local/bin/ctl init --env-file /opt/monok8s/config/cluster.env >>/var/log/monok8s/bootstrap.log 2>&1 &
+# shellcheck source=/dev/null
+. "$MIGRATIONS_LIB"
+
+if [ "$slot_changed" -eq 1 ]; then
+	monok8s_cleanup_runtime_state
+fi
+
+K8S_MINOR="$(monok8s_detect_k8s_minor || true)"
+if [ -n "$K8S_MINOR" ]; then
+	monok8s_run_migration_dir \
+	"/usr/lib/monok8s/migrations.d/k8s/$K8S_MINOR" \
+	"$MIGRATION_STATE_DIR/k8s/$K8S_MINOR"
+fi
+
+/usr/local/bin/ctl init --env-file "$CONFIG_DIR/cluster.env" >>/var/log/monok8s/bootstrap.log 2>&1 &

alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+monok8s_detect_k8s_minor() {
+	local env_file=${1:-}
+	local version major_minor
+
+	version=$(/usr/local/bin/ctl version -k 2>/dev/null || true)
+
+	[ -n "$version" ] || return 1
+
+	version=${version#v}
+	major_minor=$(printf '%s\n' "$version" | cut -d. -f1,2)
+
+	[ -n "$major_minor" ] || return 1
+	printf '%s\n' "$major_minor"
+}
+
+monok8s_cleanup_runtime_state() {
+	echo "Cleaning runtime state"
+
+	rm -rf \
+		/var/lib/containers \
+		/var/lib/cni \
+		/var/lib/kubelet/pods \
+		/var/lib/kubelet/plugins \
+		/var/lib/kubelet/plugins_registry \
+		/var/lib/kubelet/device-plugins \
+		/run/containers \
+		/run/netns
+
+	mkdir -p \
+		/var/lib/containers \
+		/var/lib/kubelet \
+		/var/lib/cni
+}
+
+monok8s_run_migration_dir() {
+	dir=$1
+	state_dir=$2
+
+	[ -d "$dir" ] || return 0
+	mkdir -p "$state_dir"
+
+	for script in "$dir"/*.sh; do
+		[ -e "$script" ] || continue
+
+		name=$(basename "$script")
+		stamp="$state_dir/$name.done"
+
+		if [ -e "$stamp" ]; then
+			continue
+		fi
+
+		echo "Running migration: $script"
+		sh "$script"
+		: > "$stamp"
+	done
+}