diff --git a/.gitignore b/.gitignore index dca5a56..22b1c22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ clitools/bin packages/ out/ +*_gen.go *.swp diff --git a/alpine/configure-system.sh b/alpine/configure-system.sh index a4d49a5..9d931d1 100755 --- a/alpine/configure-system.sh +++ b/alpine/configure-system.sh @@ -11,5 +11,3 @@ rc-update add fancontrol boot rc-update add loopback boot rc-update add hostname boot rc-update add localmount boot -rc-update add apply-node-config default -rc-update add bootstrap-cluster default diff --git a/clitools/README b/clitools/README new file mode 100644 index 0000000..5e1e863 --- /dev/null +++ b/clitools/README @@ -0,0 +1,28 @@ +## For development workflow + +Run this on device +```bash +while true; do nc -l -p 1234 -e sh; done +``` + +Run this script on the dev machine +```bash +#!/bin/bash +make build + +SIZE=$(wc -c < ./bin/ctl-linux-aarch64-dev) + +( + echo 'base64 -d > /var/ctl <<'"'"'EOF'"'"'' + pv -s "$SIZE" < ./bin/ctl-linux-aarch64-dev | base64 + echo 'EOF' + echo 'chmod +x /var/ctl' + echo '/var/ctl create config > /var/abc.yaml' + echo "/var/ctl internal run-step $1 -c /var/abc.yaml 2>&1" +) | nc 10.0.0.10 1234 +``` + +And use it like this +```bash +./send.sh start_crio +``` diff --git a/clitools/makefile b/clitools/makefile index 7ab571c..f4ce3f3 100644 --- a/clitools/makefile +++ b/clitools/makefile @@ -1,11 +1,22 @@ VERSION ?= dev +KUBE_VERSION=v1.35.1 + BIN_DIR := bin -build: +BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go + +$(BUILDINFO_FILE): + echo 'package buildinfo' > $@ + echo '' >> $@ + echo 'const DefaultKubernetesVersion = "$(KUBE_VERSION)"' >> $@ + +build: $(BUILDINFO_FILE) mkdir -p $(BIN_DIR) GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/ctl-linux-aarch64-$(VERSION) ./cmd/ctl/ -# go build -o $(BIN_DIR)/ctl-$(VERSION) ./cmd/ctl + +build-local: + go build -o $(BIN_DIR)/ctl-$(VERSION) ./cmd/ctl run: go run ./cmd/ctl diff --git a/clitools/pkg/apis/monok8s/v1alpha1/types.go b/clitools/pkg/apis/monok8s/v1alpha1/types.go index 51ddb30..24e389f 100644 --- a/clitools/pkg/apis/monok8s/v1alpha1/types.go +++ b/clitools/pkg/apis/monok8s/v1alpha1/types.go @@ -20,8 +20,8 @@ var ( type MonoKSConfig struct { metav1.TypeMeta `json:",inline" yaml:",inline"` metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec MonoKSConfigSpec `json:"spec,omitempty" yaml:"spec,omitempty"` - Status MonoKSConfigStatus `json:"status,omitempty" yaml:"status,omitempty"` + Spec MonoKSConfigSpec `json:"spec,omitempty" yaml:"spec,omitempty"` + Status *MonoKSConfigStatus `json:"status,omitempty" yaml:"status,omitempty"` } type MonoKSConfigList struct { @@ -35,13 +35,13 @@ type MonoKSConfigSpec struct { NodeName string `json:"nodeName,omitempty" yaml:"nodeName,omitempty"` ClusterName string `json:"clusterName,omitempty" yaml:"clusterName,omitempty"` ClusterDomain string `json:"clusterDomain,omitempty" yaml:"clusterDomain,omitempty"` + ClusterRole string `json:"clusterRole,omitempty" yaml:"clusterRole,omitempty"` + InitControlPlane bool `json:"initControlPlane,omitempty" yaml:"initControlPlane,omitempty"` PodSubnet string `json:"podSubnet,omitempty" yaml:"podSubnet,omitempty"` ServiceSubnet string `json:"serviceSubnet,omitempty" yaml:"serviceSubnet,omitempty"` APIServerAdvertiseAddress string `json:"apiServerAdvertiseAddress,omitempty" yaml:"apiServerAdvertiseAddress,omitempty"` APIServerEndpoint string `json:"apiServerEndpoint,omitempty" yaml:"apiServerEndpoint,omitempty"` ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint,omitempty" yaml:"containerRuntimeEndpoint,omitempty"` - BootstrapMode string `json:"bootstrapMode,omitempty" yaml:"bootstrapMode,omitempty"` - JoinKind string `json:"joinKind,omitempty" yaml:"joinKind,omitempty"` BootstrapToken string `json:"bootstrapToken,omitempty" yaml:"bootstrapToken,omitempty"` DiscoveryTokenCACertHash string `json:"discoveryTokenCACertHash,omitempty" yaml:"discoveryTokenCACertHash,omitempty"` ControlPlaneCertKey string `json:"controlPlaneCertKey,omitempty" yaml:"controlPlaneCertKey,omitempty"` @@ -74,8 +74,8 @@ type MonoKSConfigStatus struct { type OSUpgrade struct { metav1.TypeMeta `json:",inline" yaml:",inline"` metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec OSUpgradeSpec `json:"spec,omitempty" yaml:"spec,omitempty"` - Status OSUpgradeStatus `json:"status,omitempty" yaml:"status,omitempty"` + Spec OSUpgradeSpec `json:"spec,omitempty" yaml:"spec,omitempty"` + Status *OSUpgradeStatus `json:"status,omitempty" yaml:"status,omitempty"` } type OSUpgradeList struct { diff --git a/clitools/pkg/bootstrap/registry.go b/clitools/pkg/bootstrap/registry.go index 20ffccd..964ec2d 100644 --- a/clitools/pkg/bootstrap/registry.go +++ b/clitools/pkg/bootstrap/registry.go @@ -12,6 +12,7 @@ type Registry struct { func NewRegistry(ctx *node.NodeContext) *Registry { netCfg := node.NetworkConfig{ + Hostname: ctx.Config.Spec.Network.Hostname, MgmtIface: ctx.Config.Spec.Network.ManagementIface, MgmtAddress: ctx.Config.Spec.Network.ManagementCIDR, MgmtGateway: ctx.Config.Spec.Network.ManagementGW, @@ -21,13 +22,10 @@ func NewRegistry(ctx *node.NodeContext) *Registry { return &Registry{ steps: map[string]node.Step{ - "check_prereqs": node.CheckPrereqs, "validate_network_requirements": node.ValidateNetworkRequirements, - "install_cni_if_requested": node.InstallCNIIfRequested, + "configure_default_cni": node.ConfigureDefaultCNI, "start_crio": node.StartCRIO, - "check_crio_running": node.CheckCRIORunning, "wait_for_existing_cluster_if_needed": node.WaitForExistingClusterIfNeeded, - "decide_bootstrap_action": node.DecideBootstrapAction, "check_required_images": node.CheckRequiredImages, "generate_kubeadm_config": node.GenerateKubeadmConfig, "run_kubeadm_init": node.RunKubeadmInit, @@ -35,6 +33,7 @@ func NewRegistry(ctx *node.NodeContext) *Registry { "apply_local_node_metadata_if_possible": node.ApplyLocalNodeMetadataIfPossible, "allow_single_node_scheduling": node.AllowSingleNodeScheduling, "ensure_ip_forward": node.EnsureIPForward, + "configure_hostname": node.ConfigureHostname(netCfg), "configure_mgmt_interface": node.ConfigureMgmtInterface(netCfg), "configure_dns": node.ConfigureDNS(netCfg), "set_hostname_if_needed": node.SetHostnameIfNeeded, diff --git a/clitools/pkg/bootstrap/runner.go b/clitools/pkg/bootstrap/runner.go index f2a3c59..677c8d3 100644 --- a/clitools/pkg/bootstrap/runner.go +++ b/clitools/pkg/bootstrap/runner.go @@ -16,8 +16,8 @@ type Runner struct { func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner { runnerCfg := system.RunnerConfig{} nctx := &node.NodeContext{ - Config: cfg, - System: system.NewRunner(runnerCfg), + Config: cfg, + SystemRunner: system.NewRunner(runnerCfg), } return &Runner{ NodeCtx: nctx, @@ -27,11 +27,13 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner { func (r *Runner) Init(ctx context.Context) error { for _, name := range []string{ - "check_prereqs", + "configure_hostname", + "configure_dns", + "configure_mgmt_interface", "validate_network_requirements", - "install_cni_if_requested", + "configure_default_cni", "start_crio", - "check_crio_running", + "check_container_images", "wait_for_existing_cluster_if_needed", "decide_bootstrap_action", "check_required_images", diff --git a/clitools/pkg/buildinfo/README b/clitools/pkg/buildinfo/README new file mode 100644 index 0000000..06a3d45 --- /dev/null +++ b/clitools/pkg/buildinfo/README @@ -0,0 +1 @@ +Use `make build` to generate the files. Do not modify. diff --git a/clitools/pkg/cmd/create/create.go b/clitools/pkg/cmd/create/create.go index bc78ef9..22a4441 100644 --- a/clitools/pkg/cmd/create/create.go +++ b/clitools/pkg/cmd/create/create.go @@ -2,9 +2,9 @@ package create import ( "fmt" - - "undecided.project/monok8s/pkg/templates" "github.com/spf13/cobra" + + render "undecided.project/monok8s/pkg/render" ) func NewCmdCreate() *cobra.Command { @@ -14,7 +14,11 @@ func NewCmdCreate() *cobra.Command { Use: "config", Short: "Print a MonoKSConfig template", RunE: func(cmd *cobra.Command, _ []string) error { - _, err := fmt.Fprint(cmd.OutOrStdout(), templates.MonoKSConfigYAML) + out, err := render.RenderMonoKSConfig() + if err != nil { + return err + } + _, err = fmt.Fprint(cmd.OutOrStdout(), out) return err }, }, @@ -22,7 +26,11 @@ func NewCmdCreate() *cobra.Command { Use: "osupgrade", Short: "Print an OSUpgrade template", RunE: func(cmd *cobra.Command, _ []string) error { - _, err := fmt.Fprint(cmd.OutOrStdout(), templates.OSUpgradeYAML) + out, err := render.RenderOSUpgrade() + if err != nil { + return err + } + _, err = fmt.Fprint(cmd.OutOrStdout(), out) return err }, }, diff --git a/clitools/pkg/cmd/root/root.go b/clitools/pkg/cmd/root/root.go index c1e1541..0055d9d 100644 --- a/clitools/pkg/cmd/root/root.go +++ b/clitools/pkg/cmd/root/root.go @@ -3,15 +3,16 @@ package root import ( "flag" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" agentcmd "undecided.project/monok8s/pkg/cmd/agent" applycmd "undecided.project/monok8s/pkg/cmd/apply" checkconfigcmd "undecided.project/monok8s/pkg/cmd/checkconfig" createcmd "undecided.project/monok8s/pkg/cmd/create" initcmd "undecided.project/monok8s/pkg/cmd/initcmd" internalcmd "undecided.project/monok8s/pkg/cmd/internal" - "github.com/spf13/cobra" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/klog/v2" + versioncmd "undecided.project/monok8s/pkg/cmd/version" ) func NewRootCmd() *cobra.Command { @@ -30,6 +31,7 @@ func NewRootCmd() *cobra.Command { flags.AddFlags(cmd.PersistentFlags()) cmd.AddCommand( + versioncmd.NewCmdVersion(), initcmd.NewCmdInit(flags), checkconfigcmd.NewCmdCheckConfig(), createcmd.NewCmdCreate(), diff --git a/clitools/pkg/cmd/version/version.go b/clitools/pkg/cmd/version/version.go new file mode 100644 index 0000000..a351a64 --- /dev/null +++ b/clitools/pkg/cmd/version/version.go @@ -0,0 +1,22 @@ +package apply + +import ( + "fmt" + + "github.com/spf13/cobra" + + buildInfo "undecided.project/monok8s/pkg/buildinfo" +) + +func NewCmdVersion() *cobra.Command { + cmd := &cobra.Command{ + Use: "version", + Short: "Print the version information", + RunE: func(cmd *cobra.Command, _ []string) error { + + _, err := fmt.Fprintln(cmd.OutOrStdout(), buildInfo.Version) + return err + }, + } + return cmd +} diff --git a/clitools/pkg/config/config.go b/clitools/pkg/config/config.go index f944e60..cd7503a 100644 --- a/clitools/pkg/config/config.go +++ b/clitools/pkg/config/config.go @@ -6,8 +6,8 @@ import ( "os" "strings" - monov1alpha1 "undecided.project/monok8s/pkg/apis/monok8s/v1alpha1" "gopkg.in/yaml.v3" + monov1alpha1 "undecided.project/monok8s/pkg/apis/monok8s/v1alpha1" ) const EnvVar = "MONOKSCONFIG" @@ -62,11 +62,8 @@ func ApplyDefaults(cfg *monov1alpha1.MonoKSConfig) { if cfg.Spec.ContainerRuntimeEndpoint == "" { cfg.Spec.ContainerRuntimeEndpoint = "unix:///var/run/crio/crio.sock" } - if cfg.Spec.BootstrapMode == "" { - cfg.Spec.BootstrapMode = "init" - } - if cfg.Spec.JoinKind == "" { - cfg.Spec.JoinKind = "worker" + if cfg.Spec.ClusterRole == "" { + cfg.Spec.ClusterRole = "control-plane" } if cfg.Spec.CNIPlugin == "" { cfg.Spec.CNIPlugin = "none" @@ -102,30 +99,26 @@ func Validate(cfg *monov1alpha1.MonoKSConfig) error { if !strings.Contains(cfg.Spec.Network.ManagementCIDR, "/") { problems = append(problems, "spec.network.managementCIDR must include a CIDR prefix") } - if cfg.Spec.BootstrapMode != "init" && cfg.Spec.BootstrapMode != "join" { - problems = append(problems, "spec.bootstrapMode must be init or join") - } - if cfg.Spec.JoinKind != "worker" && cfg.Spec.JoinKind != "control-plane" { - problems = append(problems, "spec.joinKind must be worker or control-plane") + if cfg.Spec.ClusterRole != "control-plane" && cfg.Spec.ClusterRole != "worker" { + problems = append(problems, "spec.clusterRole can either be control-plane or worker") } for _, ns := range cfg.Spec.Network.DNSNameservers { if ns == "10.96.0.10" { problems = append(problems, "spec.network.dnsNameservers must not include cluster DNS service IP 10.96.0.10") } } - if cfg.Spec.BootstrapMode == "join" { + if cfg.Spec.ClusterRole == "worker" { if cfg.Spec.APIServerEndpoint == "" { - problems = append(problems, "spec.apiServerEndpoint is required for join mode") + problems = append(problems, "spec.apiServerEndpoint is required to join a cluster") } if cfg.Spec.BootstrapToken == "" { - problems = append(problems, "spec.bootstrapToken is required for join mode") + problems = append(problems, "spec.bootstrapToken is required to join a cluster") } if cfg.Spec.DiscoveryTokenCACertHash == "" { - problems = append(problems, "spec.discoveryTokenCACertHash is required for join mode") - } - if cfg.Spec.JoinKind == "control-plane" && cfg.Spec.ControlPlaneCertKey == "" { - problems = append(problems, "spec.controlPlaneCertKey is required for control-plane join") + problems = append(problems, "spec.discoveryTokenCACertHash is required to join a cluster") } + } else if !cfg.Spec.InitControlPlane && cfg.Spec.ControlPlaneCertKey == "" { + problems = append(problems, "spec.controlPlaneCertKey is required for control-plane join") } if len(problems) > 0 { return errors.New(strings.Join(problems, "; ")) diff --git a/clitools/pkg/node/context.go b/clitools/pkg/node/context.go index d308b4f..4a19a4d 100644 --- a/clitools/pkg/node/context.go +++ b/clitools/pkg/node/context.go @@ -9,7 +9,7 @@ import ( type NodeContext struct { Config *monov1alpha1.MonoKSConfig - System *system.Runner + SystemRunner *system.Runner } type Step func(context.Context, *NodeContext) error diff --git a/clitools/pkg/node/crio.go b/clitools/pkg/node/crio.go index a99f261..9a451c8 100644 --- a/clitools/pkg/node/crio.go +++ b/clitools/pkg/node/crio.go @@ -2,21 +2,61 @@ package node import ( "context" + "fmt" + "os" + "strings" "k8s.io/klog/v2" + system "undecided.project/monok8s/pkg/system" ) -func InstallCNIIfRequested(context.Context, *NodeContext) error { - klog.Info("install_cni_if_requested: TODO implement bridge/none CNI toggling") +func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error { + _ = ctx + + const ( + cniDir = "/etc/cni/net.d" + enabledPath = cniDir + "/10-crio-bridge.conflist" + disabledPath = cniDir + "/10-crio-bridge.conflist.disabled" + ) + + plugin := strings.TrimSpace(n.Config.Spec.CNIPlugin) + + switch plugin { + case "none": + // Fail hard if we cannot ensure the default bridge CNI is disabled. + if _, err := os.Stat(enabledPath); err == nil { + if err := os.Rename(enabledPath, disabledPath); err != nil { + return fmt.Errorf("disable default CRI-O bridge CNI: %w", err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat %s: %w", enabledPath, err) + } + + klog.Infof("Default CRI-O bridge CNI disabled") + return nil + + case "bridge": + fallthrough + case "default": + // Fail soft. User can still install or provide their own CNI. + if _, err := os.Stat(disabledPath); err == nil { + if err := os.Rename(disabledPath, enabledPath); err != nil { + klog.Warningf("failed enabling default CRI-O bridge CNI: %v", err) + return nil + } + } else if !os.IsNotExist(err) { + klog.Warningf("failed stating %s while enabling default CRI-O bridge CNI: %v", disabledPath, err) + return nil + } + + klog.Infof("Default CRI-O bridge CNI enabled") + return nil + + } + klog.Infof("unsupported CNIPlugin: %q", plugin) return nil } -func StartCRIO(context.Context, *NodeContext) error { - klog.Info("start_crio: TODO implement rc-service crio start") - return nil -} - -func CheckCRIORunning(context.Context, *NodeContext) error { - klog.Info("check_crio_running: TODO implement crictl readiness checks") - return nil +func StartCRIO(ctx context.Context, n *NodeContext) error { + return system.EnsureServiceRunning(ctx, n.SystemRunner, "crio") } diff --git a/clitools/pkg/node/kubeadm.go b/clitools/pkg/node/kubeadm.go index 1a3b757..b8a8805 100644 --- a/clitools/pkg/node/kubeadm.go +++ b/clitools/pkg/node/kubeadm.go @@ -1,36 +1,87 @@ package node import ( - "context" + "context" + "fmt" + "strings" - "k8s.io/klog/v2" + "k8s.io/klog/v2" ) func WaitForExistingClusterIfNeeded(context.Context, *NodeContext) error { - klog.Info("wait_for_existing_cluster_if_needed: TODO implement kubelet/admin.conf waits") - return nil + klog.Info("wait_for_existing_cluster_if_needed: TODO implement kubelet/admin.conf waits") + return nil } -func CheckRequiredImages(context.Context, *NodeContext) error { - klog.Info("check_required_images: TODO implement kubeadm image list + crictl image presence") - return nil + +func CheckRequiredImages(ctx context.Context, n *NodeContext) error { + if n.Config.Spec.SkipImageCheck { + klog.Infof("skipping image check (skipImageCheck=true)") + return nil + } + + k8sVersion := strings.TrimSpace(n.Config.Spec.KubernetesVersion) + if k8sVersion == "" { + return fmt.Errorf("kubernetesVersion is required") + } + + klog.Infof("checking required Kubernetes images for %s...", k8sVersion) + + result, err := n.SystemRunner.Run(ctx, + "kubeadm", "config", "images", "list", + "--kubernetes-version", k8sVersion, + ) + if err != nil { + return fmt.Errorf("list required Kubernetes images for %s: %w", k8sVersion, err) + } + + var missing []string + for _, img := range strings.Fields(result.Stdout) { + if err := checkImagePresent(ctx, n, img); err != nil { + klog.Errorf("MISSING image: %s", img) + missing = append(missing, img) + continue + } + klog.Infof("found image: %s", img) + } + + if len(missing) > 0 { + return fmt.Errorf("preload the Kubernetes images before bootstrapping; missing: %s", strings.Join(missing, ", ")) + } + + klog.Infof("all required images are present") + return nil +} + +func checkImagePresent(ctx context.Context, n *NodeContext, image string) error { + image = strings.TrimSpace(image) + if image == "" { + return fmt.Errorf("image is required") + } + + // crictl inspecti exits non-zero when the image is absent. + _, err := n.SystemRunner.Run(ctx, "crictl", "inspecti", image) + if err != nil { + return fmt.Errorf("image %q not present: %w", image, err) + } + return nil } func GenerateKubeadmConfig(context.Context, *NodeContext) error { - klog.Info("generate_kubeadm_config: TODO render kubeadm v1beta4 config from MonoKSConfig") - return nil + klog.Info("generate_kubeadm_config: TODO render kubeadm v1beta4 config from MonoKSConfig") + return nil } func RunKubeadmInit(context.Context, *NodeContext) error { - klog.Info("run_kubeadm_init: TODO implement kubeadm init --config ") - return nil + klog.Info("run_kubeadm_init: TODO implement kubeadm init --config ") + return nil } func RunKubeadmUpgradeApply(context.Context, *NodeContext) error { - klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply") - return nil + klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply") + return nil } func RunKubeadmJoin(context.Context, *NodeContext) error { - klog.Info("run_kubeadm_join: TODO implement kubeadm join") - return nil + klog.Info("run_kubeadm_join: TODO implement kubeadm join") + return nil } func RunKubeadmUpgradeNode(context.Context, *NodeContext) error { - klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node") - return nil + klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node") + return nil } diff --git a/clitools/pkg/node/network.go b/clitools/pkg/node/network.go index 4f40943..b2a538d 100644 --- a/clitools/pkg/node/network.go +++ b/clitools/pkg/node/network.go @@ -12,6 +12,7 @@ import ( ) type NetworkConfig struct { + Hostname string MgmtIface string MgmtAddress string MgmtGateway string @@ -48,11 +49,11 @@ func ConfigureMgmtInterface(cfg NetworkConfig) Step { } } - if _, err := nctx.System.Run(ctx, "ip", "link", "show", "dev", cfg.MgmtIface); err != nil { + if _, err := nctx.SystemRunner.Run(ctx, "ip", "link", "show", "dev", cfg.MgmtIface); err != nil { return fmt.Errorf("interface not found: %s: %w", cfg.MgmtIface, err) } - if _, err := nctx.System.Run(ctx, "ip", "link", "set", "dev", cfg.MgmtIface, "up"); err != nil { + if _, err := nctx.SystemRunner.Run(ctx, "ip", "link", "set", "dev", cfg.MgmtIface, "up"); err != nil { return fmt.Errorf("failed to bring up interface %s: %w", cfg.MgmtIface, err) } @@ -64,13 +65,13 @@ func ConfigureMgmtInterface(cfg NetworkConfig) Step { if hasAddr { klog.Infof("address already present on %s: %s", cfg.MgmtIface, wantCIDR) } else { - if _, err := nctx.System.Run(ctx, "ip", "addr", "add", wantCIDR, "dev", cfg.MgmtIface); err != nil { + if _, err := nctx.SystemRunner.Run(ctx, "ip", "addr", "add", wantCIDR, "dev", cfg.MgmtIface); err != nil { return fmt.Errorf("failed assigning %s to %s: %w", wantCIDR, cfg.MgmtIface, err) } } if gw := strings.TrimSpace(cfg.MgmtGateway); gw != "" { - if _, err := nctx.System.Run(ctx, "ip", "route", "replace", "default", "via", gw, "dev", cfg.MgmtIface); err != nil { + if _, err := nctx.SystemRunner.Run(ctx, "ip", "route", "replace", "default", "via", gw, "dev", cfg.MgmtIface); err != nil { return fmt.Errorf("failed setting default route via %s dev %s: %w", gw, cfg.MgmtIface, err) } } @@ -85,7 +86,44 @@ func maskSize(m net.IPMask) int { } func EnsureIPForward(ctx context.Context, n *NodeContext) error { - return system.EnsureSysctl(ctx, n.System, "net.ipv4.ip_forward", "1") + return system.EnsureSysctl(ctx, n.SystemRunner, "net.ipv4.ip_forward", "1") +} + +func ConfigureHostname(cfg NetworkConfig) Step { + return func(context.Context, *NodeContext) error { + want := strings.TrimSpace(cfg.Hostname) + if want == "" { + return fmt.Errorf("hostname is required") + } + + current, err := os.Hostname() + if err != nil { + current = "" + } + + if current == want { + return nil + } + + if err := system.SetHostname(want); err != nil { + return fmt.Errorf("set hostname to %q: %w", want, err) + } + + if err := os.WriteFile("/etc/hostname", []byte(want+"\n"), 0o644); err != nil { + return fmt.Errorf("write /etc/hostname: %w", err) + } + + current, err = os.Hostname() + if err != nil { + current = "" + } + + if current != want { + return fmt.Errorf("Unable to set hostname: %q", want) + } + + return nil + } } func ConfigureDNS(cfg NetworkConfig) Step { @@ -159,7 +197,7 @@ func ConfigureDNS(cfg NetworkConfig) Step { } func interfaceHasIPv4(ctx context.Context, nctx *NodeContext, iface, wantIP string) (bool, error) { - res, err := nctx.System.Run(ctx, "ip", "-o", "-4", "addr", "show", "dev", iface) + res, err := nctx.SystemRunner.Run(ctx, "ip", "-o", "-4", "addr", "show", "dev", iface) if err != nil { return false, err } diff --git a/clitools/pkg/node/prereqs.go b/clitools/pkg/node/prereqs.go index b6599c4..bd9b0ea 100644 --- a/clitools/pkg/node/prereqs.go +++ b/clitools/pkg/node/prereqs.go @@ -1,27 +1,112 @@ package node import ( - "context" + "context" + "fmt" + "net" + "strings" + "time" - "k8s.io/klog/v2" + "k8s.io/klog/v2" ) -func CheckPrereqs(context.Context, *NodeContext) error { - klog.Info("check_prereqs: TODO implement command discovery and runtime validation") - return nil -} +func ValidateNetworkRequirements(ctx context.Context, nct *NodeContext) error { + requireLocalIP := func(wantedIP string) error { + wantedIP = strings.TrimSpace(wantedIP) + if wantedIP == "" { + return fmt.Errorf("API server advertise address is required") + } -func ValidateNetworkRequirements(context.Context, *NodeContext) error { - klog.Info("validate_network_requirements: TODO implement local IP and API reachability checks") - return nil + ip := net.ParseIP(wantedIP) + if ip == nil { + return fmt.Errorf("invalid API server advertise address %q", wantedIP) + } + + ifaces, err := net.Interfaces() + if err != nil { + return fmt.Errorf("list interfaces: %w", err) + } + + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + continue + } + for _, addr := range addrs { + var got net.IP + switch v := addr.(type) { + case *net.IPNet: + got = v.IP + case *net.IPAddr: + got = v.IP + } + if got != nil && got.Equal(ip) { + return nil + } + } + } + + return fmt.Errorf("required local IP is not present on any interface: %s", wantedIP) + } + + checkAPIServerReachable := func(endpoint string) error { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return fmt.Errorf("API server endpoint is required") + } + + host, port, err := net.SplitHostPort(endpoint) + if err != nil { + return fmt.Errorf("invalid API server endpoint %q: %w", endpoint, err) + } + if strings.TrimSpace(host) == "" || strings.TrimSpace(port) == "" { + return fmt.Errorf("invalid API server endpoint %q", endpoint) + } + + klog.Infof("checking API server reachability: %s:%s", host, port) + + var lastErr error + for i := 0; i < 20; i++ { + d := net.Dialer{Timeout: 1 * time.Second} + conn, err := d.DialContext(ctx, "tcp", endpoint) + if err == nil { + _ = conn.Close() + klog.Infof("API server is reachable") + return nil + } + lastErr = err + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(1 * time.Second): + } + } + + return fmt.Errorf("cannot reach API server at %s: %w", endpoint, lastErr) + } + + cfg := nct.Config.Spec + switch strings.TrimSpace(cfg.ClusterRole) { + case "control-plane": + if err := requireLocalIP(cfg.APIServerAdvertiseAddress); err != nil { + return err + } + case "worker": + if err := requireLocalIP(cfg.APIServerAdvertiseAddress); err != nil { + return err + } + if err := checkAPIServerReachable(cfg.APIServerEndpoint); err != nil { + return err + } + default: + return fmt.Errorf("Incorrect ClusterRole: %s", cfg.ClusterRole) + } + + return nil } func CheckUpgradePrereqs(context.Context, *NodeContext) error { - klog.Info("check_upgrade_prereqs: TODO implement kubeadm version / skew checks") - return nil -} - -func DecideBootstrapAction(_ context.Context, nctx *NodeContext) error { - klog.InfoS("decide_bootstrap_action", "bootstrapMode", nctx.Config.Spec.BootstrapMode, "joinKind", nctx.Config.Spec.JoinKind) - return nil + klog.Info("check_upgrade_prereqs: TODO implement kubeadm version / skew checks") + return nil } diff --git a/clitools/pkg/render/monoks.go b/clitools/pkg/render/monoks.go new file mode 100644 index 0000000..c4f7645 --- /dev/null +++ b/clitools/pkg/render/monoks.go @@ -0,0 +1,55 @@ +package templates + +import ( + "bytes" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + + "undecided.project/monok8s/pkg/scheme" + tmpl "undecided.project/monok8s/pkg/templates" +) + +func RenderMonoKSConfig() (string, error) { + cfg := tmpl.DefaultMonoKSConfig() + + s := runtime.NewScheme() + if err := scheme.AddToScheme(s); err != nil { + return "", err + } + + serializer := json.NewYAMLSerializer( + json.DefaultMetaFactory, + s, + s, + ) + + var buf bytes.Buffer + if err := serializer.Encode(&cfg, &buf); err != nil { + return "", err + } + + return buf.String(), nil +} + +func RenderOSUpgrade() (string, error) { + cfg := tmpl.DefaultOSUpgrade() + + s := runtime.NewScheme() + if err := scheme.AddToScheme(s); err != nil { + return "", err + } + + serializer := json.NewYAMLSerializer( + json.DefaultMetaFactory, + s, + s, + ) + + var buf bytes.Buffer + if err := serializer.Encode(&cfg, &buf); err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/clitools/pkg/scheme/v1alpha1.go b/clitools/pkg/scheme/v1alpha1.go new file mode 100644 index 0000000..8054680 --- /dev/null +++ b/clitools/pkg/scheme/v1alpha1.go @@ -0,0 +1,27 @@ +package scheme + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "undecided.project/monok8s/pkg/apis/monok8s/v1alpha1" +) + +var ( + GroupVersion = schema.GroupVersion{ + Group: "monok8s.io", + Version: "v1alpha1", + } +) + +func AddToScheme(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, + &types.MonoKSConfig{}, + ) + + // Required for meta stuff + metav1.AddToGroupVersion(s, GroupVersion) + + return nil +} diff --git a/clitools/pkg/system/helpers.go b/clitools/pkg/system/helpers.go index 6b13973..b350fca 100644 --- a/clitools/pkg/system/helpers.go +++ b/clitools/pkg/system/helpers.go @@ -5,15 +5,19 @@ import ( "fmt" "os" "strings" + + "k8s.io/klog/v2" ) const DefaultSecond = 1_000_000_000 func EnsureServiceRunning(ctx context.Context, r *Runner, svc string) error { + if _, err := r.Run(ctx, " rc-service", svc, "status"); err == nil { return nil } + klog.Infof("Starting service: %q", svc) _, err := r.RunRetry(ctx, RetryOptions{ Attempts: 3, Delay: 2 * DefaultSecond, diff --git a/clitools/pkg/system/hostname_linux.go b/clitools/pkg/system/hostname_linux.go new file mode 100644 index 0000000..d95c248 --- /dev/null +++ b/clitools/pkg/system/hostname_linux.go @@ -0,0 +1,12 @@ +//go:build linux + +package system + +import "golang.org/x/sys/unix" + +func SetHostname(hostname string) error { + if hostname == "" { + return nil + } + return unix.Sethostname([]byte(hostname)) +} diff --git a/clitools/pkg/system/hostname_stub.go b/clitools/pkg/system/hostname_stub.go new file mode 100644 index 0000000..3188277 --- /dev/null +++ b/clitools/pkg/system/hostname_stub.go @@ -0,0 +1,8 @@ +//go:build !linux + +package system + +func SetHostname(hostname string) error { + // intentionally a no-op + return nil +} diff --git a/clitools/pkg/templates/templates.go b/clitools/pkg/templates/templates.go index a1708f2..ac9cae0 100644 --- a/clitools/pkg/templates/templates.go +++ b/clitools/pkg/templates/templates.go @@ -1,54 +1,93 @@ package templates -const MonoKSConfigYAML = `apiVersion: monok8s.io/v1alpha1 -kind: MonoKSConfig -metadata: - name: example - namespace: kube-system -spec: - kubernetesVersion: v1.35.3 - nodeName: monok8s-master-1 - clusterName: monok8s - clusterDomain: cluster.local - podSubnet: 10.244.0.0/16 - serviceSubnet: 10.96.0.0/12 - apiServerAdvertiseAddress: 10.0.0.10 - apiServerEndpoint: 10.0.0.10:6443 - containerRuntimeEndpoint: unix:///var/run/crio/crio.sock - bootstrapMode: init - joinKind: worker - cniPlugin: none - allowSchedulingOnControlPlane: true - skipImageCheck: false - kubeProxyNodePortAddresses: - - primary - subjectAltNames: - - 10.0.0.10 - nodeLabels: - node-role.kubernetes.io/control-plane: "" - nodeAnnotations: {} - network: - hostname: monok8s-master-1 - managementIface: eth0 - managementCIDR: 10.0.0.10/24 - managementGateway: 10.0.0.1 - dnsNameservers: - - 1.1.1.1 - - 8.8.8.8 - dnsSearchDomains: - - lan -` +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "undecided.project/monok8s/pkg/apis/monok8s/v1alpha1" + buildinfo "undecided.project/monok8s/pkg/buildinfo" +) -const OSUpgradeYAML = `apiVersion: monok8s.io/v1alpha1 -kind: OSUpgrade -metadata: - name: example - namespace: kube-system -spec: - version: v0.0.1 - imageURL: https://example.invalid/images/monok8s-v0.0.1.img.zst - targetPartition: B - nodeSelector: - - monok8s-master-1 - force: false -` +func DefaultMonoKSConfig() types.MonoKSConfig { + return types.MonoKSConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "monok8s.io/v1alpha1", + Kind: "MonoKSConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "kube-system", + }, + Spec: types.MonoKSConfigSpec{ + KubernetesVersion: buildinfo.Version, + NodeName: "monok8s-master-1", + + ClusterRole: "control-plane", + InitControlPlane: true, + + ClusterName: "monok8s", + ClusterDomain: "cluster.local", + + PodSubnet: "10.244.0.0/16", + ServiceSubnet: "10.96.0.0/12", + + APIServerAdvertiseAddress: "10.0.0.10", + APIServerEndpoint: "10.0.0.10:6443", + + ContainerRuntimeEndpoint: "unix:///var/run/crio/crio.sock", + + CNIPlugin: "default", + + AllowSchedulingOnControlPlane: true, + SkipImageCheck: false, + + KubeProxyNodePortAddresses: []string{ + "primary", + }, + + SubjectAltNames: []string{ + "10.0.0.10", + }, + + NodeLabels: map[string]string{ + "node-role.kubernetes.io/control-plane": "", + }, + + NodeAnnotations: map[string]string{}, + + Network: types.NetworkSpec{ + Hostname: "monok8s-master-1", + ManagementIface: "eth0", + ManagementCIDR: "10.0.0.10/24", + ManagementGW: "10.0.0.1", + DNSNameservers: []string{ + "1.1.1.1", + "8.8.8.8", + }, + DNSSearchDomains: []string{ + "lan", + }, + }, + }, + } +} + +func DefaultOSUpgrade() types.OSUpgrade { + return types.OSUpgrade{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "monok8s.io/v1alpha1", + Kind: "OSUpgrade", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "kube-system", + }, + Spec: types.OSUpgradeSpec{ + Version: "v0.0.1", + ImageURL: "https://example.invalid/images/monok8s-v0.0.1.img.zst", + TargetPartition: "B", + NodeSelector: []string{ + "monok8s-master-1", + }, + Force: false, + }, + } +} diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile index e0c0f36..2cdb6a5 100644 --- a/docker/alpine.Dockerfile +++ b/docker/alpine.Dockerfile @@ -3,9 +3,11 @@ ARG DOCKER_IMAGE_ROOT=monok8s FROM --platform=$BUILDPLATFORM ${DOCKER_IMAGE_ROOT}/build-base:${TAG} AS build-base +ARG TAG ARG ALPINE_ARCH ARG ALPINE_VER ARG CRIO_VERSION +ARG KUBE_VERSION ARG DEVICE_TREE_TARGET RUN mkdir -p "/out/rootfs" @@ -20,7 +22,10 @@ COPY out/Image.gz ./ RUN tar -xf alpine.tar.gz -C "/out/rootfs" RUN mkdir -p /out/rootfs/usr/local/bin/ -COPY packages/kubernetes/* /out/rootfs/usr/local/bin/ +COPY packages/kubernetes/kubelet-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubelet +COPY packages/kubernetes/kubeadm-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubeadm +COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubectl +COPY clitools/bin/ctl-linux-${ALPINE_ARCH}-${TAG} /out/rootfs/usr/local/bin/ctl RUN chmod +x /out/rootfs/usr/local/bin/* COPY alpine/rootfs-extra ./rootfs-extra diff --git a/makefile b/makefile index ed0568c..f7d26e2 100644 --- a/makefile +++ b/makefile @@ -13,9 +13,9 @@ NXP_TAR := $(PACKAGES_DIR)/$(NXP_VERSION).tar.gz CRIO_TAR := $(PACKAGES_DIR)/$(CRIO_VERSION).tar.gz # Kubernetes components -KUBELET_BIN := $(PACKAGES_DIR)/kubernetes/kubelet -KUBEADM_BIN := $(PACKAGES_DIR)/kubernetes/kubeadm -KUBECTL_BIN := $(PACKAGES_DIR)/kubernetes/kubectl +KUBELET_BIN := $(PACKAGES_DIR)/kubernetes/kubelet-$(KUBE_VERSION) +KUBEADM_BIN := $(PACKAGES_DIR)/kubernetes/kubeadm-$(KUBE_VERSION) +KUBECTL_BIN := $(PACKAGES_DIR)/kubernetes/kubectl-$(KUBE_VERSION) CONFIGS_DIR := configs SCRIPTS_DIR := scripts @@ -31,6 +31,7 @@ RELEASE_IMAGE := $(OUT_DIR)/monok8s-$(TAG).img.gz KERNEL_IMAGE := $(OUT_DIR)/Image.gz BUILD_BASE_STAMP := $(OUT_DIR)/.build-base-$(TAG).stamp +CLITOOLS_BIN := bin/ctl-linux-$(ARCH)-$(TAG) ALPINE_SERIES := $(word 1,$(subst ., ,$(ALPINE_VER))).$(word 2,$(subst ., ,$(ALPINE_VER))) @@ -46,6 +47,8 @@ ALPINE_SRCS := $(shell find alpine -type f 2>/dev/null) INITRAMFS_SRCS := $(shell find initramfs -type f 2>/dev/null) KERNEL_SRCS := $(shell find kernel-build -type f 2>/dev/null) +CLITOOLS_SRCS := $(shell find clitools -type f 2>/dev/null) + BUILD_BASE_DEPS := \ docker/build-base.Dockerfile \ build.env \ @@ -77,11 +80,15 @@ ITB_DEPS := \ build.env \ makefile +CLITOOLS := \ + $(CLITOOLS_SRCS) + RELEASE_DEPS := \ $(BUILD_BASE_STAMP) \ $(BUILD_INFO_FILE) \ $(BOARD_ITB) \ $(ALPINE_TAR) \ + $(CLITOOLS_BIN) \ $(CRIO_TAR) \ $(KUBELET_BIN) \ $(KUBEADM_BIN) \ @@ -91,12 +98,6 @@ RELEASE_DEPS := \ build.env \ makefile -CLITOOLS := \ - clitools/pkg \ - clitools/cmd \ - clitools/go.mod \ - clitools/go.sum - # ---- Directory creation ------------------------------------------------------ $(PACKAGES_DIR): @@ -174,6 +175,9 @@ $(INITRAMFS): $(INITRAMFS_DEPS) | $(OUT_DIR) --output type=local,dest=./$(OUT_DIR) . test -f $@ +$(CLITOOLS_BIN): $(CLITOOLS_SRCS) + $(MAKE) -C clitools + $(BOARD_ITB): $(ITB_DEPS) | $(OUT_DIR) docker build \ -f docker/itb.Dockerfile \ @@ -191,6 +195,7 @@ $(RELEASE_IMAGE): $(RELEASE_DEPS) | $(OUT_DIR) --build-arg TAG=$(TAG) \ --build-arg ALPINE_ARCH=$(ALPINE_ARCH) \ --build-arg ALPINE_VER=$(ALPINE_VER) \ + --build-arg KUBE_VERSION=$(KUBE_VERSION) \ --build-arg CRIO_VERSION=$(CRIO_VERSION) \ --build-arg DEVICE_TREE_TARGET=$(DEVICE_TREE_TARGET) \ -t $(DOCKER_IMAGE_ROOT)/buildenv-alpine:$(TAG) . @@ -263,9 +268,7 @@ kernel: $(KERNEL_IMAGE) initramfs: $(INITRAMFS) itb: $(BOARD_ITB) build-base: $(BUILD_BASE_STAMP) - -clitools: $(CLITOOLS) - $(MAKE) -C clitools +clitools: $(CLITOOLS_BIN) clean: rm -f \ @@ -274,7 +277,7 @@ clean: $(INITRAMFS) \ $(BOARD_ITB) \ $(RELEASE_IMAGE) \ - $(CLITOOLS) + $(CLITOOLS_BIN) distclean: clean rm -rf $(OUT_DIR)