From 7d20a2e92049d207edfcd483450a6f73650b01c43d734650867d6c4e147c569e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=9F=E9=85=8C=20=E9=B5=AC=E5=85=84?= Date: Thu, 26 Mar 2026 19:33:15 +0800 Subject: [PATCH] More kmods for kubelet --- README.md | 2 +- alpine/install-packages.sh | 7 + alpine/rootfs-extra/etc/fstab | 3 + .../rootfs-extra/etc/init.d/apply-node-config | 15 +- .../rootfs-extra/etc/init.d/bootstrap-cluster | 17 +- .../opt/scripts/apply-node-config.sh | 0 .../opt/scripts/bootstrap-cluster.sh | 183 ++++++++++++++++-- kernel-extra.config | 74 +++++-- 8 files changed, 255 insertions(+), 46 deletions(-) mode change 100644 => 100755 alpine/rootfs-extra/opt/scripts/apply-node-config.sh diff --git a/README.md b/README.md index 5cad91a..66c24b5 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ make itb # Builds out/board.itb (contains the kernel and the initramfs) ``` ## Architecture - - A/B deployment + - A/B deployment (So we can just do kubectl apply upgrade.yaml with version jumps. Agent will walk through it automatically) - Read-only OS ## Upgrade process diff --git a/alpine/install-packages.sh b/alpine/install-packages.sh index 5fe0d75..3eb98f7 100755 --- a/alpine/install-packages.sh +++ b/alpine/install-packages.sh @@ -46,5 +46,12 @@ mkdir -p /etc/kubernetes/manifests mkdir -p /run/crun mkdir -p /run/runc +# Kubernetes legacy things that need to exists but have no use for us +mkdir -p /usr/libexec/kubernetes/kubelet-plugins/volume/exec +chmod 0755 /usr/libexec/kubernetes \ + /usr/libexec/kubernetes/kubelet-plugins \ + /usr/libexec/kubernetes/kubelet-plugins/volume \ + /usr/libexec/kubernetes/kubelet-plugins/volume/exec + touch /var/log/crio/crio.log touch /var/log/crio/kubelet.log diff --git a/alpine/rootfs-extra/etc/fstab b/alpine/rootfs-extra/etc/fstab index 055fce5..1bd6ff5 100644 --- a/alpine/rootfs-extra/etc/fstab +++ b/alpine/rootfs-extra/etc/fstab @@ -5,3 +5,6 @@ PARTLABEL=data /data ext4 rw,noatime,nodiratime tmpfs /run tmpfs defaults,nosuid,nodev,mode=0755 0 0 tmpfs /tmp tmpfs defaults,nosuid,nodev,noexec,mode=1777 0 0 + +# Do this on Prodution +# tmpfs /var/log tmpfs defaults,nosuid,nodev,noexec,mode=0755 0 0 diff --git a/alpine/rootfs-extra/etc/init.d/apply-node-config b/alpine/rootfs-extra/etc/init.d/apply-node-config index 546b958..ccded75 100755 --- a/alpine/rootfs-extra/etc/init.d/apply-node-config +++ b/alpine/rootfs-extra/etc/init.d/apply-node-config @@ -5,12 +5,19 @@ export PATH="/usr/local/bin:/usr/local/sbin:$PATH" name="Apply node config" description="Apply node configurations using node.env from /opt/monok8s/config" -command="/opt/monok8s/scripts/apply-node-config.sh" -command_background="no" +command="/opt/scripts/apply-node-config.sh" -output_log="/var/log/monok8s/apply-node-config.log" -error_log="/var/log/monok8s/apply-node-config.err" +LOG_DIR="/var/log/monok8s" +LOG_FILE="$LOG_DIR/apply-node-config.log" depend() { need localmount } + +start() { + checkpath --directory "$LOG_DIR" + + ebegin "Applying node config" + "$command" >>"$LOG_FILE" 2>&1 + eend $? +} diff --git a/alpine/rootfs-extra/etc/init.d/bootstrap-cluster b/alpine/rootfs-extra/etc/init.d/bootstrap-cluster index d9893db..d3de5b7 100755 --- a/alpine/rootfs-extra/etc/init.d/bootstrap-cluster +++ b/alpine/rootfs-extra/etc/init.d/bootstrap-cluster @@ -3,15 +3,22 @@ export PATH="/usr/local/bin:/usr/local/sbin:$PATH" name="Bootstrap cluster" -description="Apply node configurations using node.env from /opt/monok8s/config" +description="Apply cluster configurations using node.env from /opt/monok8s/config" -command="/opt/monok8s/scripts/bootstrap-cluster.sh" -command_background="no" +command="/opt/scripts/bootstrap-cluster.sh" -output_log="/var/log/monok8s/bootstrap.log" -error_log="/var/log/monok8s/bootstrap.err" +LOG_DIR="/var/log/monok8s" +LOG_FILE="$LOG_DIR/bootstrap.log" depend() { need apply-node-config use net } + +start() { + checkpath --directory "$LOG_DIR" + + ebegin "Applying cluster config" + "$command" >>"$LOG_FILE" 2>&1 & + eend $? +} diff --git a/alpine/rootfs-extra/opt/scripts/apply-node-config.sh b/alpine/rootfs-extra/opt/scripts/apply-node-config.sh old mode 100644 new mode 100755 diff --git a/alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh b/alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh index 9752bd2..2e12a44 100755 --- a/alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh +++ b/alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh @@ -78,6 +78,151 @@ validate_config() { esac } +normalize_version() { + # strip leading "v" + echo "${1#v}" +} + +version_major_minor() { + normalize_version "$1" | awk -F. '{ print $1 "." $2 }' +} + +version_eq() { + [ "$(normalize_version "$1")" = "$(normalize_version "$2")" ] +} + +version_lt() { + [ "$(printf '%s\n%s\n' "$(normalize_version "$1")" "$(normalize_version "$2")" | sort -V | head -n1)" != "$(normalize_version "$2")" ] +} + +version_gt() { + [ "$(printf '%s\n%s\n' "$(normalize_version "$1")" "$(normalize_version "$2")" | sort -V | tail -n1)" = "$(normalize_version "$1")" ] \ + && ! version_eq "$1" "$2" +} + +minor_diff() { + a="$(version_major_minor "$1")" + b="$(version_major_minor "$2")" + a_major="${a%.*}" + a_minor="${a#*.}" + b_major="${b%.*}" + b_minor="${b#*.}" + + [ "$a_major" = "$b_major" ] || fail "major version change unsupported here: $1 -> $2" + echo $((b_minor - a_minor)) +} + +get_kubeadm_binary_version() { + kubeadm version -o short +} + +get_cluster_server_version() { + kubectl --kubeconfig /etc/kubernetes/admin.conf version -o yaml \ + | awk ' + $1 == "serverVersion:" { in_server=1; next } + in_server && $1 == "gitVersion:" { print $2; exit } + ' +} + +get_api_server_version_from_kubelet_kubeconfig() { + kubectl --kubeconfig /etc/kubernetes/kubelet.conf version -o yaml \ + | awk ' + $1 == "serverVersion:" { in_server=1; next } + in_server && $1 == "gitVersion:" { print $2; exit } + ' +} + +validate_target_matches_local_binaries() { + kubeadm_ver="$(get_kubeadm_binary_version)" + + if ! version_eq "$kubeadm_ver" "$KUBERNETES_VERSION"; then + fail "kubeadm binary version ($kubeadm_ver) does not match target KUBERNETES_VERSION ($KUBERNETES_VERSION)" + fi +} + +decide_bootstrap_action() { + case "$BOOTSTRAP_MODE" in + init) + if [ -f /etc/kubernetes/admin.conf ]; then + BOOTSTRAP_ACTION="upgrade-control-plane" + else + BOOTSTRAP_ACTION="init" + fi + ;; + join) + if [ -f /etc/kubernetes/kubelet.conf ]; then + BOOTSTRAP_ACTION="upgrade-node" + else + BOOTSTRAP_ACTION="join" + fi + ;; + *) + fail "unsupported BOOTSTRAP_MODE: $BOOTSTRAP_MODE" + ;; + esac + + log "selected bootstrap action: $BOOTSTRAP_ACTION" +} + +validate_upgrade_path() { + current="$1" + target="$2" + + if version_eq "$current" "$target"; then + log "cluster is already at target version: $target" + return 0 + fi + + if version_gt "$current" "$target"; then + fail "downgrade is not supported: current=$current target=$target" + fi + + diff="$(minor_diff "$current" "$target")" + case "$diff" in + 0|1) + ;; + *) + fail "unsupported upgrade path: current=$current target=$target (minor skip too large)" + ;; + esac +} + +check_upgrade_prereqs() { + validate_target_matches_local_binaries +} + +run_kubeadm_upgrade_apply() { + current_version="$(get_cluster_server_version)" + log "current control-plane version: $current_version" + log "target control-plane version: $KUBERNETES_VERSION" + + validate_upgrade_path "$current_version" "$KUBERNETES_VERSION" + + if version_eq "$current_version" "$KUBERNETES_VERSION"; then + log "control-plane already at target version; skipping kubeadm upgrade apply" + return 0 + fi + + log "running kubeadm upgrade plan..." + kubeadm upgrade plan "$KUBERNETES_VERSION" + + log "running kubeadm upgrade apply..." + kubeadm upgrade apply -y "$KUBERNETES_VERSION" +} + +run_kubeadm_upgrade_node() { + cluster_version="$(get_api_server_version_from_kubelet_kubeconfig)" + log "cluster/control-plane version visible from this node: $cluster_version" + log "target node version: $KUBERNETES_VERSION" + + if ! version_eq "$cluster_version" "$KUBERNETES_VERSION"; then + fail "control-plane version ($cluster_version) does not match target ($KUBERNETES_VERSION); upgrade control-plane first" + fi + + log "running kubeadm upgrade node..." + kubeadm upgrade node +} + check_prereqs() { need_cmd kubeadm need_cmd kubelet @@ -289,23 +434,6 @@ validate_network_requirements() { esac } -setup_local_kubectl() { - kube_dir="${KUBECONFIG_USER_HOME}/.kube" - log "setting up local kubectl config in ${kube_dir}/config..." - - mkdir -p "$kube_dir" - cp /etc/kubernetes/admin.conf "${kube_dir}/config" - chmod 600 "${kube_dir}/config" - - if [ "$KUBECONFIG_USER_HOME" = "/root" ]; then - mkdir -p /etc/profile.d - cat > /etc/profile.d/kubeconfig.sh <<'EOF' -export KUBECONFIG=/root/.kube/config -EOF - chmod 644 /etc/profile.d/kubeconfig.sh - fi -} - wait_for_node() { log "waiting for node registration: $NODE_NAME" @@ -409,25 +537,40 @@ main() { check_prereqs validate_network_requirements - check_not_already_bootstrapped + decide_bootstrap_action install_cni_if_requested start_crio check_crio_running - case "$BOOTSTRAP_MODE" in + case "$BOOTSTRAP_ACTION" in init) check_required_images generate_kubeadm_config run_kubeadm_init rc-service kubelet restart - setup_local_kubectl + apply_local_node_metadata_if_possible + allow_single_node_scheduling + ;; + upgrade-control-plane) + check_upgrade_prereqs + check_required_images + run_kubeadm_upgrade_apply + rc-service kubelet restart apply_local_node_metadata_if_possible allow_single_node_scheduling ;; join) run_kubeadm_join ;; + upgrade-node) + check_upgrade_prereqs + run_kubeadm_upgrade_node + rc-service kubelet restart + ;; + *) + fail "unsupported BOOTSTRAP_ACTION: $BOOTSTRAP_ACTION" + ;; esac print_next_steps diff --git a/kernel-extra.config b/kernel-extra.config index 989bc0c..efd4499 100644 --- a/kernel-extra.config +++ b/kernel-extra.config @@ -146,9 +146,12 @@ CONFIG_NF_NAT=y CONFIG_NF_TABLES=y # nftables framework. Modern Linux packet filtering backend. -CONFIG_NFT_CT=m +CONFIG_NFT_CT=y # nftables conntrack expressions. +CONFIG_NFT_COUNTER=y +# nftables packet/byte counters + CONFIG_NFT_CHAIN_NAT=y # nftables NAT chain support. @@ -161,50 +164,89 @@ CONFIG_NFT_REDIR=y CONFIG_NFT_NAT=y # nftables NAT support. -CONFIG_NF_NAT_IPV4=m +CONFIG_NF_NAT_IPV4=y # IPv4 NAT helper support. Some kernels still expose this separately. -CONFIG_NF_NAT_IPV6=m +CONFIG_NF_NAT_IPV6=y # IPv6 NAT helper support. -CONFIG_IP_NF_IPTABLES=m +CONFIG_NF_CT_NETLINK=y +# userspace netlink access to the conntrack table; kube-proxy uses this for conntrack listing/cleanup + +CONFIG_NF_CT_NETLINK_TIMEOUT=y +# userspace netlink support for conntrack timeout objects + +CONFIG_NF_CT_NETLINK_HELPER=y +# userspace netlink support for conntrack helper objects + +CONFIG_IP_NF_IPTABLES=y # iptables compatibility for IPv4. Still useful because lots of CNI/plugin code # still expects iptables even on nft-backed systems. -CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_NAT=y # IPv4 NAT support for iptables compatibility. -CONFIG_IP6_NF_IPTABLES=m +CONFIG_IP6_NF_IPTABLES=y # ip6tables compatibility. +CONFIG_IP6_NF_FILTER=y +# IPv6 "filter" table (same as above but for IPv6) + +CONFIG_NF_REJECT_IPV4=y +# core IPv4 reject logic used by netfilter/iptables/nftables + +CONFIG_NFT_REJECT=y +# nftables equivalent of REJECT (needed for nf_tables backend compatibility) + +CONFIG_IP_NF_FILTER=y +# IPv4 "filter" table (INPUT/FORWARD/OUTPUT chains for iptables) + +CONFIG_IP_NF_TARGET_REJECT=y +# IPv4-specific REJECT target for legacy iptables + +CONFIG_IP6_NF_TARGET_REJECT=y +# IPv6-specific REJECT target for legacy iptables + CONFIG_IP_SET=m # IP sets. Useful for some network policies / firewalling toolchains. -CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_NETLINK_ACCT=y +# netfilter accounting subsystem used for nfacct-based kube-proxy metrics + +CONFIG_NETFILTER_XT_MATCH_NFACCT=y +# iptables nfacct match that hooks rules into the netfilter accounting subsystem + +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y # xtables match for address types. Often used in iptables rules. -CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_TARGET_REJECT=y +# iptables REJECT target (actively reject packets instead of silently dropping) + +CONFIG_NETFILTER_XT_MATCH_COMMENT=y # Allows comments in iptables rules. Not critical, but harmless and useful. -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y # xtables conntrack matching. -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +# iptables "statistic" match used for probabilistic packet matching / load balancing + +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y # Match multiple ports in one rule. -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=y # Useful for TCP MSS clamping in some network paths. -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y # iptables MASQUERADE target. Very commonly needed for pod outbound NAT. -CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=y # Redirect target. -CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_MARK=y # Packet marking support. Useful for advanced networking/routing rules. -CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_CT=y # Connection tracking target for xtables. # Optional. Good only if you know you need transparent proxying. @@ -230,7 +272,7 @@ CONFIG_BRIDGE_NETFILTER=y # Optional / version-dependent: # Some kernels expose additional ebtables/bridge netfilter pieces separately. # Keep this if your kernel has it, but don't panic if it doesn't. -CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_NF_EBTABLES=y # Bridge filtering via ebtables compatibility. Sometimes useful, not always critical.