penguin/monok8s
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added migrations.d

Browse Source
This commit is contained in:
斟酌 鵬兄
2026-04-16 10:44:42 +08:00
parent f1a7074528
commit 65c643d7a2
11 changed files with 368 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -26,13 +26,11 @@ https://docs.mono.si/gateway-development-kit/getting-started
### Kubernetes
* OSUpgrade
* [x] Control Plane - kubeadm upgrade apply
* [ ] Worker node - kubeadm upgrade node
* Upgrade chain
* [x] 1.33.3 -> 1.33.10
* [ ] 1.33.10 -> 1.34.1
* [ ] 1.34.1 -> 1.34.6
* [ ] 1.34.6 -> 1.35.1
* [ ] 1.35.1 -> 1.35.3
* [x] 1.33.10 -> 1.34.6
* [x] 1.34.6 -> 1.35.3
* [ ] Worker node - kubeadm upgrade node
* CNI
* [x] default bridge-cni
* [ ] Cilium

10
alpine/build-rootfs.sh
View File

@@ -17,6 +17,7 @@ mkdir -p \
"$ROOTFS/build" \
"$ROOTFS/var/cache/apk" \
"$ROOTFS/usr/lib/monok8s/crds" \
"$ROOTFS/usr/lib/monok8s/migrations.d/k8s" \
"$ROOTFS/opt/monok8s/config"
mount --bind /var/cache/apk "$ROOTFS/var/cache/apk"
@@ -30,6 +31,15 @@ cp /etc/resolv.conf "$ROOTFS/etc/resolv.conf"
cp /build/crio.tar.gz "$ROOTFS/build/"
cp /build/crds/*.yaml "$ROOTFS/usr/lib/monok8s/crds"
KUBE_MINOR=$(printf '%s\n' "$KUBE_VERSION" | sed -E 's/^v?([0-9]+\.[0-9]+).*/\1/')
MIG_SRC="/build/migrations/k8s/$KUBE_MINOR"
MIG_DST="$ROOTFS/usr/lib/monok8s/migrations.d/k8s/$KUBE_MINOR"
if [ -d "$MIG_SRC" ]; then
mkdir -p "$MIG_DST"
cp -a "$MIG_SRC"/. "$MIG_DST"/
fi
chroot "$ROOTFS" /bin/sh -c "ln -s /var/cache/apk /etc/apk/cache"
# chroot "$ROOTFS" /bin/sh -c "apk update"
chroot "$ROOTFS" /bin/sh -c "apk add bash curl"

35
alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh Executable file
View File

@@ -0,0 +1,35 @@
#!/bin/sh
set -eu
# Kubernetes removed the kubelet flag:
# --pod-infra-container-image
#
# Timeline:
# - Deprecated before v1.27
# - Removed in newer kubelet versions (>=1.27+)
# - kubeadm may still write it into:
# /var/lib/kubelet/kubeadm-flags.env
#
# This causes kubelet to fail with:
# "unknown flag: --pod-infra-container-image"
#
# References:
# - https://github.com/kubernetes/kubeadm/issues/3281
# - https://github.com/kubernetes/kubernetes/pull/122739
# - https://kubernetes.io/blog/2022/04/07/upcoming-changes-in-kubernetes-1-24/
#
# Root cause:
# - Sandbox (pause) image is now managed by CRI (containerd/CRI-O),
# not kubelet flags.
#
# Fix:
# - Strip the flag from kubeadm-flags.env during upgrade
FILE=/var/lib/kubelet/kubeadm-flags.env
[ -f "$FILE" ] || exit 0
grep -q -- '--pod-infra-container-image=' "$FILE" || exit 0
sed -i 's/ --pod-infra-container-image=[^"]*//g' "$FILE"
echo "Removed deprecated kubelet flag --pod-infra-container-image from $FILE"

39
alpine/rootfs-extra/etc/local.d/monok8s.start
View File

@@ -5,22 +5,37 @@ exec >>/var/log/monok8s/boot.log 2>&1
export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
MIGRATIONS_LIB=/usr/lib/monok8s/lib/migrations.sh
CONFIG_DIR=/opt/monok8s/config
BOOT_STATE=/run/monok8s/boot-state.env
BOOTPART_FILE="$CONFIG_DIR/.bootpart"
MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state"
mkdir -p /dev/hugepages
mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages
echo 640 > /proc/sys/vm/nr_hugepages
echo 256 > /proc/sys/vm/nr_hugepages
CUR=$(grep '^BOOT_PART=' /run/monok8s/boot-state.env | cut -d= -f2-)
LAST=$(cat /opt/monok8s/config/.bootpart 2>/dev/null || true)
CUR=$(grep '^BOOT_PART=' "$BOOT_STATE" | cut -d= -f2-)
LAST=$(cat "$BOOTPART_FILE" 2>/dev/null || true)
slot_changed=0
if [ "$CUR" != "$LAST" ]; then
echo "Slot changed ($LAST -> $CUR), cleaning runtime state"
rm -rf /var/lib/containers \
/var/lib/kubelet/pods \
/var/lib/kubelet/plugins \
/var/lib/kubelet/device-plugins
mkdir -p /var/lib/containers /var/lib/kubelet
slot_changed=1
echo "Slot changed ($LAST -> $CUR)"
fi
/usr/local/bin/ctl init --env-file /opt/monok8s/config/cluster.env >>/var/log/monok8s/bootstrap.log 2>&1 &
# shellcheck source=/dev/null
. "$MIGRATIONS_LIB"
if [ "$slot_changed" -eq 1 ]; then
monok8s_cleanup_runtime_state
fi
K8S_MINOR="$(monok8s_detect_k8s_minor || true)"
if [ -n "$K8S_MINOR" ]; then
monok8s_run_migration_dir \
"/usr/lib/monok8s/migrations.d/k8s/$K8S_MINOR" \
"$MIGRATION_STATE_DIR/k8s/$K8S_MINOR"
fi
/usr/local/bin/ctl init --env-file "$CONFIG_DIR/cluster.env" >>/var/log/monok8s/bootstrap.log 2>&1 &

58
alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/bin/sh
monok8s_detect_k8s_minor() {
local env_file=${1:-}
local version major_minor
version=$(/usr/local/bin/ctl version -k 2>/dev/null || true)
[ -n "$version" ] || return 1
version=${version#v}
major_minor=$(printf '%s\n' "$version" | cut -d. -f1,2)
[ -n "$major_minor" ] || return 1
printf '%s\n' "$major_minor"
}
monok8s_cleanup_runtime_state() {
echo "Cleaning runtime state"
rm -rf \
/var/lib/containers \
/var/lib/cni \
/var/lib/kubelet/pods \
/var/lib/kubelet/plugins \
/var/lib/kubelet/plugins_registry \
/var/lib/kubelet/device-plugins \
/run/containers \
/run/netns
mkdir -p \
/var/lib/containers \
/var/lib/kubelet \
/var/lib/cni
}
monok8s_run_migration_dir() {
dir=$1
state_dir=$2
[ -d "$dir" ] || return 0
mkdir -p "$state_dir"
for script in "$dir"/*.sh; do
[ -e "$script" ] || continue
name=$(basename "$script")
stamp="$state_dir/$name.done"
if [ -e "$stamp" ]; then
continue
fi
echo "Running migration: $script"
sh "$script"
: > "$stamp"
done
}

64
clitools/pkg/cmd/version/version.go
View File

@@ -1,22 +1,76 @@
package apply
package version
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
buildInfo "example.com/monok8s/pkg/buildinfo"
buildinfo "example.com/monok8s/pkg/buildinfo"
)
type versionInfo struct {
Version string `json:"version"`
GitRevision string `json:"gitRevision"`
Timestamp string `json:"timestamp"`
KubeVersion string `json:"kubernetesVersion"`
}
func NewCmdVersion() *cobra.Command {
var (
shortOutput bool
jsonOutput bool
kubernetesOutput bool
)
cmd := &cobra.Command{
Use: "version",
Short: "Print the version information",
Short: "Print version information",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
info := versionInfo{
Version: buildinfo.Version,
GitRevision: buildinfo.GitRevision,
Timestamp: buildinfo.Timestamp,
KubeVersion: buildinfo.KubeVersion,
}
_, err := fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s %s(%s)", buildInfo.Version, buildInfo.GitRevision, buildInfo.Timestamp))
return err
out := cmd.OutOrStdout()
switch {
case jsonOutput:
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
return enc.Encode(info)
case kubernetesOutput:
_, err := fmt.Fprintln(out, info.KubeVersion)
return err
case shortOutput:
_, err := fmt.Fprintln(out, info.Version)
return err
default:
_, err := fmt.Fprintf(
out,
"Version: %s\nGit commit: %s\nBuilt at: %s\nKubernetes: %s\n",
info.Version,
info.GitRevision,
info.Timestamp,
info.KubeVersion,
)
return err
}
},
}
flags := cmd.Flags()
flags.BoolVar(&shortOutput, "short", false, "Show only the application version")
flags.BoolVar(&jsonOutput, "json", false, "Show version information as JSON")
flags.BoolVarP(&kubernetesOutput, "kubernetes", "k", false, "Show only the Kubernetes version this binary was built for")
cmd.MarkFlagsMutuallyExclusive("short", "json", "kubernetes")
return cmd
}

3
clitools/pkg/controller/osimage/write_throttle_other.go
View File

@@ -15,4 +15,5 @@ func newNoopAdaptiveWriteController() *adaptiveWriteController {
}
func (c *adaptiveWriteController) Wait(ctx context.Context, n int) error { return nil }
func (c *adaptiveWriteController) ObserveWrite(n int, dur interface{}) {}
func (c *adaptiveWriteController) ObserveWrite(n int) {}
func (c *adaptiveWriteController) ObserveSync() {}

21
clitools/pkg/controller/osupgrade/planner.go
View File

@@ -224,9 +224,9 @@ func calculatePath(current, target string, available []string) ([]string, error)
add(latestCurMinor)
}
// Step 2: walk each intermediate minor using the lowest available patch in that minor.
// Step 2: walk each intermediate minor using the latest available patch in that minor.
for minor := cur.Minor + 1; minor < tgt.Minor; minor++ {
bridge, ok := lowestPatchInMinor(versions, cur.Major, minor)
bridge, ok := latestAnyPatchInMinor(versions, cur.Major, minor)
if !ok {
return nil, fmt.Errorf("no available bridge version for v%d.%d.x", cur.Major, minor)
}
@@ -239,6 +239,23 @@ func calculatePath(current, target string, available []string) ([]string, error)
return versionsToStrings(path), nil
}
func latestAnyPatchInMinor(versions []Version, major, minor int) (Version, bool) {
var found Version
ok := false
for _, v := range versions {
if v.Major != major || v.Minor != minor {
continue
}
if !ok || found.Compare(v) < 0 {
found = v
ok = true
}
}
return found, ok
}
func parseAndSortVersions(raw []string) ([]Version, error) {
out := make([]Version, 0, len(raw))
seen := map[string]struct{}{}

149
clitools/pkg/controller/osupgrade/planner_test.go Normal file
View File

@@ -0,0 +1,149 @@
package osupgrade
import (
"reflect"
"testing"
)
func TestCalculatePath(t *testing.T) {
t.Parallel()
tests := []struct {
name string
current string
target string
available []string
want []string
wantErr bool
}{
{
name: "same version returns nil path",
current: "v1.34.6",
target: "v1.34.6",
available: []string{"v1.34.6"},
want: nil,
wantErr: false,
},
{
name: "same minor jumps directly to target",
current: "v1.34.1",
target: "v1.34.6",
available: []string{"v1.34.1", "v1.34.3", "v1.34.6"},
want: []string{"v1.34.6"},
wantErr: false,
},
{
name: "next minor direct jump when no current minor patch available",
current: "v1.34.6",
target: "v1.35.3",
available: []string{"v1.34.6", "v1.35.1", "v1.35.3"},
want: []string{"v1.35.3"},
wantErr: false,
},
{
name: "finish current minor then target",
current: "v1.34.1",
target: "v1.35.3",
available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"},
want: []string{"v1.34.6", "v1.35.3"},
wantErr: false,
},
{
name: "multi minor path uses latest bridge patch",
current: "v1.33.10",
target: "v1.35.3",
available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"},
want: []string{"v1.34.6", "v1.35.3"},
wantErr: false,
},
{
name: "multi minor path finishes current minor and latest bridge patch",
current: "v1.33.1",
target: "v1.35.3",
available: []string{"v1.33.5", "v1.33.9", "v1.34.1", "v1.34.6", "v1.35.3"},
want: []string{"v1.33.9", "v1.34.6", "v1.35.3"},
wantErr: false,
},
{
name: "duplicates in available are ignored",
current: "v1.33.10",
target: "v1.35.3",
available: []string{"v1.34.6", "v1.34.6", "v1.35.3", "v1.35.3"},
want: []string{"v1.34.6", "v1.35.3"},
wantErr: false,
},
{
name: "target missing returns error",
current: "v1.34.6",
target: "v1.35.3",
available: []string{"v1.34.6", "v1.35.1"},
wantErr: true,
},
{
name: "missing bridge minor returns error",
current: "v1.33.10",
target: "v1.35.3",
available: []string{"v1.35.3"},
wantErr: true,
},
{
name: "downgrade not supported",
current: "v1.35.3",
target: "v1.34.6",
available: []string{"v1.34.6", "v1.35.3"},
wantErr: true,
},
{
name: "cross major not supported",
current: "v1.35.3",
target: "v2.0.0",
available: []string{"v1.35.3", "v2.0.0"},
wantErr: true,
},
{
name: "invalid current version returns error",
current: "garbage",
target: "v1.35.3",
available: []string{"v1.35.3"},
wantErr: true,
},
{
name: "invalid target version returns error",
current: "v1.34.6",
target: "wat",
available: []string{"v1.34.6", "v1.35.3"},
wantErr: true,
},
{
name: "invalid available version returns error",
current: "v1.34.6",
target: "v1.35.3",
available: []string{"v1.34.6", "broken", "v1.35.3"},
wantErr: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := calculatePath(tt.current, tt.target, tt.available)
if tt.wantErr {
if err == nil {
t.Fatalf("expected error, got nil; path=%v", got)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("calculatePath(%q, %q, %v)\n got: %v\n want: %v",
tt.current, tt.target, tt.available, got, tt.want)
}
})
}
}

6
clitools/pkg/node/fs.go
View File

@@ -6,6 +6,8 @@ import (
"os"
"path/filepath"
"k8s.io/klog/v2"
monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
"example.com/monok8s/pkg/controller/osimage"
)
@@ -26,7 +28,7 @@ func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error {
gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
if err := os.Remove(gateFile); err != nil {
return fmt.Errorf("relate control gate: %w", err)
return fmt.Errorf("release control gate: %w", err)
}
return WriteLastState(ctx, nctx)
@@ -46,6 +48,8 @@ func WriteLastState(ctx context.Context, nctx *NodeContext) error {
return fmt.Errorf("BOOT_PART missing")
}
klog.Infof("Writing last state: %+v", bootPart)
tmp := stBootPart + ".tmp"
if err := os.WriteFile(tmp, []byte(bootPart+"\n"), 0o644); err != nil {
return err

1
docker/alpine.Dockerfile
View File

@@ -31,6 +31,7 @@ COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubec
# COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/
# COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/
COPY alpine/rootfs-extra ./rootfs-extra
COPY alpine/migrations ./migrations
COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh
COPY alpine/*.sh /
RUN chmod +x /out/rootfs/usr/local/bin/*