More kmods for kubelet

README.md

@@ -76,7 +76,7 @@ make itb        # Builds out/board.itb (contains the kernel and the initramfs)
 ```
 
 ## Architecture
- - A/B deployment
+ - A/B deployment (So we can just do kubectl apply upgrade.yaml with version jumps. Agent will walk through it automatically)
  - Read-only OS
 
 ## Upgrade process

alpine/install-packages.sh

@@ -46,5 +46,12 @@ mkdir -p /etc/kubernetes/manifests
 mkdir -p /run/crun
 mkdir -p /run/runc
 
+# Kubernetes legacy things that need to exists but have no use for us
+mkdir -p /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+chmod 0755 /usr/libexec/kubernetes \
+            /usr/libexec/kubernetes/kubelet-plugins \
+            /usr/libexec/kubernetes/kubelet-plugins/volume \
+            /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+
 touch /var/log/crio/crio.log
 touch /var/log/crio/kubelet.log

alpine/rootfs-extra/etc/fstab

@@ -5,3 +5,6 @@ PARTLABEL=data    /data                ext4  rw,noatime,nodiratime
 
 tmpfs             /run                 tmpfs defaults,nosuid,nodev,mode=0755           0  0
 tmpfs             /tmp                 tmpfs defaults,nosuid,nodev,noexec,mode=1777    0  0
+
+# Do this on Prodution
+# tmpfs /var/log tmpfs defaults,nosuid,nodev,noexec,mode=0755 0 0

alpine/rootfs-extra/etc/init.d/apply-node-config

@@ -5,12 +5,19 @@ export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
 name="Apply node config"
 description="Apply node configurations using node.env from /opt/monok8s/config"
 
-command="/opt/monok8s/scripts/apply-node-config.sh"
+command="/opt/scripts/apply-node-config.sh"
-command_background="no"
 
-output_log="/var/log/monok8s/apply-node-config.log"
+LOG_DIR="/var/log/monok8s"
-error_log="/var/log/monok8s/apply-node-config.err"
+LOG_FILE="$LOG_DIR/apply-node-config.log"
 
 depend() {
     need localmount
 }
+
+start() {
+    checkpath --directory "$LOG_DIR"
+
+    ebegin "Applying node config"
+    "$command" >>"$LOG_FILE" 2>&1
+    eend $?
+}

alpine/rootfs-extra/etc/init.d/bootstrap-cluster

@@ -3,15 +3,22 @@
 export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
 
 name="Bootstrap cluster"
-description="Apply node configurations using node.env from /opt/monok8s/config"
+description="Apply cluster configurations using node.env from /opt/monok8s/config"
 
-command="/opt/monok8s/scripts/bootstrap-cluster.sh"
+command="/opt/scripts/bootstrap-cluster.sh"
-command_background="no"
 
-output_log="/var/log/monok8s/bootstrap.log"
+LOG_DIR="/var/log/monok8s"
-error_log="/var/log/monok8s/bootstrap.err"
+LOG_FILE="$LOG_DIR/bootstrap.log"
 
 depend() {
     need apply-node-config
     use net
 }
+
+start() {
+    checkpath --directory "$LOG_DIR"
+
+    ebegin "Applying cluster config"
+    "$command" >>"$LOG_FILE" 2>&1 &
+    eend $?
+}

alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh

@@ -78,6 +78,151 @@ validate_config() {
     esac
 }
 
+normalize_version() {
+    # strip leading "v"
+    echo "${1#v}"
+}
+
+version_major_minor() {
+    normalize_version "$1" | awk -F. '{ print $1 "." $2 }'
+}
+
+version_eq() {
+    [ "$(normalize_version "$1")" = "$(normalize_version "$2")" ]
+}
+
+version_lt() {
+    [ "$(printf '%s\n%s\n' "$(normalize_version "$1")" "$(normalize_version "$2")" | sort -V | head -n1)" != "$(normalize_version "$2")" ]
+}
+
+version_gt() {
+    [ "$(printf '%s\n%s\n' "$(normalize_version "$1")" "$(normalize_version "$2")" | sort -V | tail -n1)" = "$(normalize_version "$1")" ] \
+        && ! version_eq "$1" "$2"
+}
+
+minor_diff() {
+    a="$(version_major_minor "$1")"
+    b="$(version_major_minor "$2")"
+    a_major="${a%.*}"
+    a_minor="${a#*.}"
+    b_major="${b%.*}"
+    b_minor="${b#*.}"
+
+    [ "$a_major" = "$b_major" ] || fail "major version change unsupported here: $1 -> $2"
+    echo $((b_minor - a_minor))
+}
+
+get_kubeadm_binary_version() {
+    kubeadm version -o short
+}
+
+get_cluster_server_version() {
+    kubectl --kubeconfig /etc/kubernetes/admin.conf version -o yaml \
+        | awk '
+            $1 == "serverVersion:" { in_server=1; next }
+            in_server && $1 == "gitVersion:" { print $2; exit }
+        '
+}
+
+get_api_server_version_from_kubelet_kubeconfig() {
+    kubectl --kubeconfig /etc/kubernetes/kubelet.conf version -o yaml \
+        | awk '
+            $1 == "serverVersion:" { in_server=1; next }
+            in_server && $1 == "gitVersion:" { print $2; exit }
+        '
+}
+
+validate_target_matches_local_binaries() {
+    kubeadm_ver="$(get_kubeadm_binary_version)"
+
+    if ! version_eq "$kubeadm_ver" "$KUBERNETES_VERSION"; then
+        fail "kubeadm binary version ($kubeadm_ver) does not match target KUBERNETES_VERSION ($KUBERNETES_VERSION)"
+    fi
+}
+
+decide_bootstrap_action() {
+    case "$BOOTSTRAP_MODE" in
+        init)
+            if [ -f /etc/kubernetes/admin.conf ]; then
+                BOOTSTRAP_ACTION="upgrade-control-plane"
+            else
+                BOOTSTRAP_ACTION="init"
+            fi
+            ;;
+        join)
+            if [ -f /etc/kubernetes/kubelet.conf ]; then
+                BOOTSTRAP_ACTION="upgrade-node"
+            else
+                BOOTSTRAP_ACTION="join"
+            fi
+            ;;
+        *)
+            fail "unsupported BOOTSTRAP_MODE: $BOOTSTRAP_MODE"
+            ;;
+    esac
+
+    log "selected bootstrap action: $BOOTSTRAP_ACTION"
+}
+
+validate_upgrade_path() {
+    current="$1"
+    target="$2"
+
+    if version_eq "$current" "$target"; then
+        log "cluster is already at target version: $target"
+        return 0
+    fi
+
+    if version_gt "$current" "$target"; then
+        fail "downgrade is not supported: current=$current target=$target"
+    fi
+
+    diff="$(minor_diff "$current" "$target")"
+    case "$diff" in
+        0|1)
+            ;;
+        *)
+            fail "unsupported upgrade path: current=$current target=$target (minor skip too large)"
+            ;;
+    esac
+}
+
+check_upgrade_prereqs() {
+    validate_target_matches_local_binaries
+}
+
+run_kubeadm_upgrade_apply() {
+    current_version="$(get_cluster_server_version)"
+    log "current control-plane version: $current_version"
+    log "target control-plane version: $KUBERNETES_VERSION"
+
+    validate_upgrade_path "$current_version" "$KUBERNETES_VERSION"
+
+    if version_eq "$current_version" "$KUBERNETES_VERSION"; then
+        log "control-plane already at target version; skipping kubeadm upgrade apply"
+        return 0
+    fi
+
+    log "running kubeadm upgrade plan..."
+    kubeadm upgrade plan "$KUBERNETES_VERSION"
+
+    log "running kubeadm upgrade apply..."
+    kubeadm upgrade apply -y "$KUBERNETES_VERSION"
+}
+
+run_kubeadm_upgrade_node() {
+    cluster_version="$(get_api_server_version_from_kubelet_kubeconfig)"
+    log "cluster/control-plane version visible from this node: $cluster_version"
+    log "target node version: $KUBERNETES_VERSION"
+
+    if ! version_eq "$cluster_version" "$KUBERNETES_VERSION"; then
+        fail "control-plane version ($cluster_version) does not match target ($KUBERNETES_VERSION); upgrade control-plane first"
+    fi
+
+    log "running kubeadm upgrade node..."
+    kubeadm upgrade node
+}
+
 check_prereqs() {
     need_cmd kubeadm
     need_cmd kubelet
@@ -289,23 +434,6 @@ validate_network_requirements() {
     esac
 }
 
-setup_local_kubectl() {
-    kube_dir="${KUBECONFIG_USER_HOME}/.kube"
-    log "setting up local kubectl config in ${kube_dir}/config..."
-
-    mkdir -p "$kube_dir"
-    cp /etc/kubernetes/admin.conf "${kube_dir}/config"
-    chmod 600 "${kube_dir}/config"
-
-    if [ "$KUBECONFIG_USER_HOME" = "/root" ]; then
-        mkdir -p /etc/profile.d
-        cat > /etc/profile.d/kubeconfig.sh <<'EOF'
-export KUBECONFIG=/root/.kube/config
-EOF
-        chmod 644 /etc/profile.d/kubeconfig.sh
-    fi
-}
-
 wait_for_node() {
     log "waiting for node registration: $NODE_NAME"
 
@@ -409,25 +537,40 @@ main() {
 
     check_prereqs
     validate_network_requirements
-    check_not_already_bootstrapped
+    decide_bootstrap_action
 
     install_cni_if_requested
     start_crio
     check_crio_running
 
-    case "$BOOTSTRAP_MODE" in
+    case "$BOOTSTRAP_ACTION" in
         init)
             check_required_images
             generate_kubeadm_config
             run_kubeadm_init
             rc-service kubelet restart
-            setup_local_kubectl
+            apply_local_node_metadata_if_possible
+            allow_single_node_scheduling
+            ;;
+        upgrade-control-plane)
+            check_upgrade_prereqs
+            check_required_images
+            run_kubeadm_upgrade_apply
+            rc-service kubelet restart
             apply_local_node_metadata_if_possible
             allow_single_node_scheduling
             ;;
         join)
             run_kubeadm_join
             ;;
+        upgrade-node)
+            check_upgrade_prereqs
+            run_kubeadm_upgrade_node
+            rc-service kubelet restart
+            ;;
+        *)
+            fail "unsupported BOOTSTRAP_ACTION: $BOOTSTRAP_ACTION"
+            ;;
     esac
 
     print_next_steps

kernel-extra.config

@@ -146,9 +146,12 @@ CONFIG_NF_NAT=y
 CONFIG_NF_TABLES=y
 # nftables framework. Modern Linux packet filtering backend.
 
-CONFIG_NFT_CT=m
+CONFIG_NFT_CT=y
 # nftables conntrack expressions.
 
+CONFIG_NFT_COUNTER=y
+# nftables packet/byte counters
+
 CONFIG_NFT_CHAIN_NAT=y
 # nftables NAT chain support.
 
@@ -161,50 +164,89 @@ CONFIG_NFT_REDIR=y
 CONFIG_NFT_NAT=y
 # nftables NAT support.
 
-CONFIG_NF_NAT_IPV4=m
+CONFIG_NF_NAT_IPV4=y
 # IPv4 NAT helper support. Some kernels still expose this separately.
 
-CONFIG_NF_NAT_IPV6=m
+CONFIG_NF_NAT_IPV6=y
 # IPv6 NAT helper support.
 
-CONFIG_IP_NF_IPTABLES=m
+CONFIG_NF_CT_NETLINK=y
+# userspace netlink access to the conntrack table; kube-proxy uses this for conntrack listing/cleanup
+
+CONFIG_NF_CT_NETLINK_TIMEOUT=y
+# userspace netlink support for conntrack timeout objects
+
+CONFIG_NF_CT_NETLINK_HELPER=y
+# userspace netlink support for conntrack helper objects
+
+CONFIG_IP_NF_IPTABLES=y
 # iptables compatibility for IPv4. Still useful because lots of CNI/plugin code
 # still expects iptables even on nft-backed systems.
 
-CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_NAT=y
 # IPv4 NAT support for iptables compatibility.
 
-CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_IPTABLES=y
 # ip6tables compatibility.
 
+CONFIG_IP6_NF_FILTER=y
+# IPv6 "filter" table (same as above but for IPv6)
+
+CONFIG_NF_REJECT_IPV4=y
+# core IPv4 reject logic used by netfilter/iptables/nftables
+
+CONFIG_NFT_REJECT=y
+# nftables equivalent of REJECT (needed for nf_tables backend compatibility)
+
+CONFIG_IP_NF_FILTER=y
+# IPv4 "filter" table (INPUT/FORWARD/OUTPUT chains for iptables)
+
+CONFIG_IP_NF_TARGET_REJECT=y
+# IPv4-specific REJECT target for legacy iptables
+
+CONFIG_IP6_NF_TARGET_REJECT=y
+# IPv6-specific REJECT target for legacy iptables
+
 CONFIG_IP_SET=m
 # IP sets. Useful for some network policies / firewalling toolchains.
 
-CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_NETLINK_ACCT=y
+# netfilter accounting subsystem used for nfacct-based kube-proxy metrics
+
+CONFIG_NETFILTER_XT_MATCH_NFACCT=y
+# iptables nfacct match that hooks rules into the netfilter accounting subsystem
+
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
 # xtables match for address types. Often used in iptables rules.
 
-CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_TARGET_REJECT=y
+# iptables REJECT target (actively reject packets instead of silently dropping)
+
+CONFIG_NETFILTER_XT_MATCH_COMMENT=y
 # Allows comments in iptables rules. Not critical, but harmless and useful.
 
-CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
 # xtables conntrack matching.
 
-CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
+# iptables "statistic" match used for probabilistic packet matching / load balancing
+
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
 # Match multiple ports in one rule.
 
-CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=y
 # Useful for TCP MSS clamping in some network paths.
 
-CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m
+CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y
 # iptables MASQUERADE target. Very commonly needed for pod outbound NAT.
 
-CONFIG_NETFILTER_XT_TARGET_REDIRECT=m
+CONFIG_NETFILTER_XT_TARGET_REDIRECT=y
 # Redirect target.
 
-CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_MARK=y
 # Packet marking support. Useful for advanced networking/routing rules.
 
-CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_CT=y
 # Connection tracking target for xtables.
 
 # Optional. Good only if you know you need transparent proxying.
@@ -230,7 +272,7 @@ CONFIG_BRIDGE_NETFILTER=y
 # Optional / version-dependent:
 # Some kernels expose additional ebtables/bridge netfilter pieces separately.
 # Keep this if your kernel has it, but don't panic if it doesn't.
-CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_NF_EBTABLES=y
 # Bridge filtering via ebtables compatibility. Sometimes useful, not always critical.