diff --git a/alpine/build-rootfs.sh b/alpine/build-rootfs.sh index 7c1de54..99933b0 100755 --- a/alpine/build-rootfs.sh +++ b/alpine/build-rootfs.sh @@ -51,7 +51,7 @@ rm -r "$ROOTFS/build" echo "##################################################### Packaging RootFS "$( du -sh "$ROOTFS/" ) IMG=output.img -SIZE=1536MB +SIZE=2048MB dd if=/dev/zero of="$IMG" bs=1 count=0 seek=$SIZE diff --git a/alpine/configure-system.sh b/alpine/configure-system.sh index e27d0d2..f7e8636 100755 --- a/alpine/configure-system.sh +++ b/alpine/configure-system.sh @@ -6,6 +6,7 @@ echo "##################################################### Install RC Services" rc-update add devfs sysinit rc-update add procfs sysinit rc-update add sysfs sysinit +rc-update add cgroups sysinit rc-update add fancontrol boot rc-update add loopback boot rc-update add hostname boot diff --git a/alpine/install-packages.sh b/alpine/install-packages.sh index 9038ab3..11855cb 100755 --- a/alpine/install-packages.sh +++ b/alpine/install-packages.sh @@ -6,8 +6,17 @@ echo "##################################################### Installing basic pac apk add alpine-base \ openrc busybox-openrc bash nftables \ lm-sensors lm-sensors-fancontrol lm-sensors-fancontrol-openrc + +# For diagnotics +apk add \ + iproute2 iproute2-ss curl bind-tools procps strace tcpdump lsof jq \ + openssl nftables conntrack-tools ethtool findmnt kmod coreutils util-linux echo '[ -x /bin/bash ] && exec /bin/bash -l' >> "/root/.profile" +# Compat layer for kubelet for now. Will look into building it myself later. If needed +apk add gcompat +kubelet --version || exit 1 + echo "##################################################### Installing CRI-O" mkdir -p /usr/local/bin @@ -32,9 +41,12 @@ mkdir -p /var/run/crio mkdir -p /var/lib/containers/storage mkdir -p /var/lib/cni mkdir -p /var/log/crio +mkdir -p /var/log/kubelet mkdir -p /etc/cni/net.d mkdir -p /opt/cni/bin +mkdir -p /etc/kubernetes/manifests mkdir -p /run/crun mkdir -p /run/runc touch /var/log/crio/crio.log +touch /var/log/crio/kubelet.log diff --git a/alpine/rootfs-extra/etc/conf.d/kubelet b/alpine/rootfs-extra/etc/conf.d/kubelet new file mode 100644 index 0000000..e756fad --- /dev/null +++ b/alpine/rootfs-extra/etc/conf.d/kubelet @@ -0,0 +1 @@ +command_args="--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=cgroupfs --config=/var/lib/kubelet/config.yaml" diff --git a/alpine/rootfs-extra/etc/crictl.yaml b/alpine/rootfs-extra/etc/crictl.yaml new file mode 100644 index 0000000..4931b37 --- /dev/null +++ b/alpine/rootfs-extra/etc/crictl.yaml @@ -0,0 +1,4 @@ +runtime-endpoint: unix:///var/run/crio/crio.sock +image-endpoint: unix:///var/run/crio/crio.sock +timeout: 10 +debug: false diff --git a/alpine/rootfs-extra/etc/fstab b/alpine/rootfs-extra/etc/fstab deleted file mode 100644 index 8217562..0000000 --- a/alpine/rootfs-extra/etc/fstab +++ /dev/null @@ -1 +0,0 @@ -none /sys/fs/cgroup cgroup2 defaults 0 0 diff --git a/alpine/rootfs-extra/etc/init.d/kubelet b/alpine/rootfs-extra/etc/init.d/kubelet new file mode 100755 index 0000000..2de8112 --- /dev/null +++ b/alpine/rootfs-extra/etc/init.d/kubelet @@ -0,0 +1,23 @@ +#!/sbin/openrc-run +# Copyright 2016-2017 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export PATH="/usr/local/bin:/usr/local/sbin:$PATH" + +supervisor=supervise-daemon +description="Kubelet, a Kubernetes node agent" + +if [ -e /var/lib/kubelet/kubeadm-flags.env ]; then + . /var/lib/kubelet/kubeadm-flags.env; +fi + +command="/usr/local/bin/kubelet" +command_args="${command_args} ${KUBELET_KUBEADM_ARGS}" +pidfile="${KUBELET_PIDFILE:-/run/${RC_SVCNAME}.pid}" +: ${output_log:=/var/log/$RC_SVCNAME/$RC_SVCNAME.log} +: ${error_log:=/var/log/$RC_SVCNAME/$RC_SVCNAME.log} + +depend() { + after crio + need cgroups crio +} diff --git a/alpine/rootfs-extra/opt/scripts/bootstrap-control-plane.sh b/alpine/rootfs-extra/opt/scripts/bootstrap-control-plane.sh new file mode 100755 index 0000000..eccddb7 --- /dev/null +++ b/alpine/rootfs-extra/opt/scripts/bootstrap-control-plane.sh @@ -0,0 +1,377 @@ +#!/bin/sh +set -eu + +CONFIG_DIR="${CONFIG_DIR:-/opt/monok8s/config}" +CLUSTER_ENV="${CONFIG_DIR}/cluster.env" +KUBEADM_CONFIG_OUT="${KUBEADM_CONFIG_OUT:-/tmp/kubeadm-init.yaml}" + +log() { + echo "[monok8s] $*" +} + +fail() { + echo "[monok8s] ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +require_file() { + [ -f "$1" ] || fail "required file not found: $1" +} + +load_config() { + require_file "$CLUSTER_ENV" + + # shellcheck disable=SC1090 + . "$CLUSTER_ENV" + + : "${KUBERNETES_VERSION:?KUBERNETES_VERSION is required}" + : "${NODE_NAME:?NODE_NAME is required}" + : "${APISERVER_ADVERTISE_ADDRESS:?APISERVER_ADVERTISE_ADDRESS is required}" + + POD_SUBNET="${POD_SUBNET:-10.244.0.0/16}" + SERVICE_SUBNET="${SERVICE_SUBNET:-10.96.0.0/12}" + CLUSTER_NAME="${CLUSTER_NAME:-monok8s}" + CLUSTER_DOMAIN="${CLUSTER_DOMAIN:-cluster.local}" + CONTAINER_RUNTIME_ENDPOINT="${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/crio/crio.sock}" + SANS="${SANS:-}" + ALLOW_SCHEDULING_ON_CONTROL_PLANE="${ALLOW_SCHEDULING_ON_CONTROL_PLANE:-yes}" + SKIP_IMAGE_CHECK="${SKIP_IMAGE_CHECK:-no}" + KUBECONFIG_USER_HOME="${KUBECONFIG_USER_HOME:-/root}" +} + +check_prereqs() { + need_cmd kubeadm + need_cmd kubelet + need_cmd kubectl + need_cmd crictl + need_cmd rc-service + need_cmd awk + need_cmd grep + need_cmd sed + need_cmd hostname + + [ -S /var/run/crio/crio.sock ] || fail "CRI-O socket not found at /var/run/crio/crio.sock" +} + +set_hostname_if_needed() { + current_hostname="$(hostname 2>/dev/null || true)" + + if [ "$current_hostname" != "$NODE_NAME" ]; then + log "setting hostname to $NODE_NAME" + + hostname "$NODE_NAME" + + mkdir -p /etc + printf '%s\n' "$NODE_NAME" > /etc/hostname + + # Keep localhost mapping sane even without DNS. + if [ -f /etc/hosts ]; then + if ! grep -Eq "[[:space:]]$NODE_NAME([[:space:]]|\$)" /etc/hosts; then + printf '127.0.0.1\tlocalhost %s\n' "$NODE_NAME" >> /etc/hosts + fi + else + cat > /etc/hosts < /proc/sys/net/ipv4/ip_forward + fi + + mkdir -p /etc/sysctl.d + cat > /etc/sysctl.d/99-monok8s.conf <<'EOF' +net.ipv4.ip_forward = 1 +EOF +} + +ensure_kubelet_binary() { + if command -v kubelet >/dev/null 2>&1; then + return 0 + fi + + if [ -x /usr/local/bin/kubelet ]; then + export PATH="/usr/local/bin:$PATH" + return 0 + fi + + if [ -x /opt/cni/bin/kubelet ]; then + export PATH="/opt/cni/bin:$PATH" + return 0 + fi + + fail "kubelet binary not found; install kubelet into /usr/local/bin or make sure PATH includes it" +} + +ensure_kubelet_service() { + if ! rc-service kubelet status >/dev/null 2>&1; then + if [ ! -f /etc/init.d/kubelet ]; then + fail "kubelet OpenRC service does not exist at /etc/init.d/kubelet" + fi + + log "starting kubelet service" + rc-service kubelet start || fail "failed to start kubelet" + fi +} + +check_crio_running() { + log "checking CRI-O status..." + + if ! rc-service crio status >/dev/null 2>&1; then + fail "crio service is not running" + fi + + if ! crictl --runtime-endpoint "$CONTAINER_RUNTIME_ENDPOINT" info >/dev/null 2>&1; then + fail "crictl cannot talk to CRI-O via $CONTAINER_RUNTIME_ENDPOINT" + fi + + log "CRI-O is up" +} + +normalize_image_ref() { + # Add docker.io/library/ for bare images like pause:3.10 if needed. + # kubeadm usually emits full refs already, so this is just defensive. + ref="$1" + case "$ref" in + */* ) echo "$ref" ;; + * ) echo "docker.io/library/$ref" ;; + esac +} + +image_present() { + wanted="$1" + + repo="${wanted%:*}" + tag="${wanted##*:}" + + crictl --runtime-endpoint "$CONTAINER_RUNTIME_ENDPOINT" images \ + | awk 'NR>1 { print $1 ":" $2 }' \ + | grep -Fx "$repo:$tag" >/dev/null 2>&1 +} + +check_required_images() { + [ "$SKIP_IMAGE_CHECK" = "yes" ] && { + log "skipping image check (SKIP_IMAGE_CHECK=yes)" + return 0 + } + + log "checking required Kubernetes images for $KUBERNETES_VERSION..." + + missing_any=0 + + for img in $(kubeadm config images list --kubernetes-version "$KUBERNETES_VERSION"); do + if image_present "$img"; then + log "found image: $img" + else + echo "[monok8s] MISSING image: $img" >&2 + missing_any=1 + fi + done + + [ "$missing_any" -eq 0 ] || fail "preload the Kubernetes images before bootstrapping" + + log "all required images are present" +} + +check_node_name() { + current_hostname="$(hostname)" + if [ "$current_hostname" != "$NODE_NAME" ]; then + log "warning: hostname is '$current_hostname' but NODE_NAME is '$NODE_NAME'" + log "kubeadm will use NODE_NAME from config" + fi +} + +check_already_initialized() { + if [ -f /etc/kubernetes/admin.conf ]; then + fail "cluster already appears initialized (/etc/kubernetes/admin.conf exists)" + fi +} + +generate_kubeadm_config() { + log "generating kubeadm config at $KUBEADM_CONFIG_OUT..." + + SAN_LINES="" + if [ -n "${SANS:-}" ]; then + old_ifs="$IFS" + IFS=',' + for san in $SANS; do + san_trimmed="$(echo "$san" | sed 's/^ *//;s/ *$//')" + [ -n "$san_trimmed" ] && SAN_LINES="${SAN_LINES} - \"${san_trimmed}\" +" + done + IFS="$old_ifs" + fi + + cat > "$KUBEADM_CONFIG_OUT" </dev/null 2>&1; then + log "API server advertise address already present: $APISERVER_ADVERTISE_ADDRESS" + return 0 + fi + + warn_dummy_interface + + if ! command -v modprobe >/dev/null 2>&1; then + fail "modprobe not found; cannot create dummy interface" + fi + + log "loading dummy kernel module..." + modprobe dummy || fail "failed to load dummy kernel module" + + if ! ip link show "$DUMMY_IFACE" >/dev/null 2>&1; then + log "creating dummy interface: $DUMMY_IFACE" + ip link add "$DUMMY_IFACE" type dummy || fail "failed to create dummy interface $DUMMY_IFACE" + else + log "dummy interface already exists: $DUMMY_IFACE" + fi + + log "assigning ${APISERVER_ADVERTISE_ADDRESS}/${DUMMY_PREFIX_LEN} to $DUMMY_IFACE" + ip addr add "${APISERVER_ADVERTISE_ADDRESS}/${DUMMY_PREFIX_LEN}" dev "$DUMMY_IFACE" 2>/dev/null || true + + log "bringing up $DUMMY_IFACE" + ip link set "$DUMMY_IFACE" up || fail "failed to bring up $DUMMY_IFACE" + + if ! ip -o addr show dev "$DUMMY_IFACE" | awk '{print $4}' | cut -d/ -f1 | grep -Fx "$APISERVER_ADVERTISE_ADDRESS" >/dev/null 2>&1; then + fail "dummy interface came up, but ${APISERVER_ADVERTISE_ADDRESS} is still not present" + fi + + log "dummy interface ready: $DUMMY_IFACE -> ${APISERVER_ADVERTISE_ADDRESS}/${DUMMY_PREFIX_LEN}" +} + +setup_local_kubectl() { + kube_dir="${KUBECONFIG_USER_HOME}/.kube" + log "setting up local kubectl config in ${kube_dir}/config..." + + mkdir -p "$kube_dir" + cp /etc/kubernetes/admin.conf "${kube_dir}/config" + chmod 600 "${kube_dir}/config" + + if [ "$KUBECONFIG_USER_HOME" = "/root" ]; then + mkdir -p /etc/profile.d + cat > /etc/profile.d/kubeconfig.sh <<'EOF' +export KUBECONFIG=/root/.kube/config +EOF + chmod 644 /etc/profile.d/kubeconfig.sh + fi +} + +allow_single_node_scheduling() { + if [ "$ALLOW_SCHEDULING_ON_CONTROL_PLANE" != "yes" ]; then + log "leaving control-plane taint in place" + return 0 + fi + + log "removing control-plane taint so this single node can schedule workloads..." + kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes "$NODE_NAME" node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true +} + +print_next_steps() { + cat < /tmp/called.txt + @grep -E '^[a-zA-Z_][a-zA-Z0-9_]*\(\)' ./alpine/rootfs-extra/opt/scripts/bootstrap-control-plane.sh \ + | sed 's/().*//' \ + | sort -u > /tmp/defined.txt + @echo "Missing functions:" + @comm -23 /tmp/called.txt /tmp/defined.txt || true + # ---- User targets ------------------------------------------------------------ release: $(RELEASE_IMAGE)