Added join step

clitools/README

@@ -26,3 +26,36 @@ And use it like this
 ```bash
 ./send.sh start_crio
 ```
+
+
+Create join token from control plane
+```
+kubeadm token create --print-join-command
+```
+
+Export the token inside the device
+```
+export DEBUG=1
+export HOSTNAME=monok8s-master-1
+export BOOTSTRAP_TOKEN=
+export TOKEN_CACERT_HASH=
+```
+
+Generate using kubectl
+```
+TOKEN_NAME=bootstrap-token-iwotl0
+
+API_SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||') && \
+TOKEN=$(kubectl -n kube-system get secret ${TOKEN_NAME} -o jsonpath='{.data.token-id}' | base64 -d).$(kubectl -n kube-system get secret ${TOKEN_NAME} -o jsonpath='{.data.token-secret}' | base64 -d) && \
+HASH=$(kubectl -n kube-public get configmap cluster-info -o jsonpath='{.data.kubeconfig}' \
+| grep 'certificate-authority-data' \
+| awk '{print $2}' \
+| base64 -d \
+| openssl x509 -pubkey -noout \
+| openssl rsa -pubin -outform der 2>/dev/null \
+| openssl dgst -sha256 -hex \
+| awk '{print "sha256:" $2}')
+echo "export API_SERVER_ENDPOINT=${API_SERVER}"
+echo "export BOOTSTRAP_TOKEN=${TOKEN}"
+echo "export TOKEN_CACERT_HASH=${HASH}"
+```

clitools/pkg/bootstrap/registry.go

@@ -24,6 +24,7 @@ func NewRegistry(ctx *node.NodeContext) *Registry {
 		steps: map[string]node.Step{
 			"AllowSingleNodeScheduling":              node.AllowSingleNodeScheduling,
 			"ApplyLocalNodeMetadataIfPossible":       node.ApplyLocalNodeMetadataIfPossible,
+			"CheckForVersionSkew":                    node.CheckForVersionSkew,
 			"CheckUpgradePrereqs":                    node.CheckUpgradePrereqs,
 			"ClassifyBootstrapAction":                node.ClassifyBootstrapAction,
 			"ConfigureDNS":                           node.ConfigureDNS(netCfg),
@@ -32,7 +33,6 @@ func NewRegistry(ctx *node.NodeContext) *Registry {
 			"ConfigureMgmtInterface":                 node.ConfigureMgmtInterface(netCfg),
 			"DetectLocalClusterState":                node.DetectLocalClusterState,
 			"EnsureIPForward":                        node.EnsureIPForward,
-			"GenerateKubeadmConfig":                  node.GenerateKubeadmConfig,
 			"PrintSummary":                           node.PrintSummary,
 			"ReconcileControlPlane":                  node.ReconcileControlPlane,
 			"ReconcileNode":                          node.ReconcileNode,

clitools/pkg/bootstrap/runner.go

@@ -86,6 +86,16 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Classify bootstrap action",
 				Desc:   "Decide whether to init, join, upgrade, or reconcile based on local state and desired version",
 			},
+			{
+				RegKey: "CheckForVersionSkew",
+				Name:   "Check for version skew",
+				Desc:   "Validate wether version satisfy the requirements againts current cluster if any",
+			},
+			{
+				RegKey: "RunKubeadmInit",
+				Name:   "Run kubeadm init",
+				Desc:   "Initialize a new Kubernetes control plane using kubeadm",
+			},
 			{
 				RegKey: "WaitForExistingClusterIfNeeded",
 				Name:   "Wait for existing cluster",
@@ -106,11 +116,6 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
 				Name:   "Run kubeadm upgrade apply",
 				Desc:   "Upgrade control plane components using kubeadm",
 			},
-			{
-				RegKey: "RunKubeadmInit",
-				Name:   "Run kubeadm init",
-				Desc:   "Initialize a new Kubernetes control plane using kubeadm",
-			},
 			{
 				RegKey: "RunKubeadmJoin",
 				Name:   "Run kubeadm join",

clitools/pkg/node/context.go

@@ -34,13 +34,17 @@ type LocalClusterState struct {
 type BootstrapAction string
 
 const (
-	BootstrapActionInitControlPlane      BootstrapAction = "init-control-plane"
+	BootstrapActionInitControlPlane   BootstrapAction = "init-control-plane"
-	BootstrapActionJoinWorker            BootstrapAction = "join-worker"
+	BootstrapActionJoinControlPlane   BootstrapAction = "join-control-plane"
-	BootstrapActionJoinControlPlane      BootstrapAction = "join-control-plane"
+	BootstrapActionManageControlPlane BootstrapAction = "manage-control-plane"
-	BootstrapActionReconcileWorker       BootstrapAction = "reconcile-worker"
+
+	BootstrapActionJoinWorker   BootstrapAction = "join-worker"
+	BootstrapActionManageWorker BootstrapAction = "manage-worker"
+
 	BootstrapActionReconcileControlPlane BootstrapAction = "reconcile-control-plane"
-	BootstrapActionUpgradeWorker         BootstrapAction = "upgrade-worker"
 	BootstrapActionUpgradeControlPlane   BootstrapAction = "upgrade-control-plane"
+	BootstrapActionReconcileWorker       BootstrapAction = "reconcile-worker"
+	BootstrapActionUpgradeWorker         BootstrapAction = "upgrade-worker"
 )
 
 type BootstrapState struct {

clitools/pkg/node/kubeadm.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 
+	types "undecided.project/monok8s/pkg/apis/monok8s/v1alpha1"
 	system "undecided.project/monok8s/pkg/system"
 )
 
@@ -114,6 +115,10 @@ func WaitForExistingClusterIfNeeded(ctx context.Context, nctx *NodeContext) erro
 }
 
 func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error {
+	if nctx.BootstrapState == nil {
+		return errors.New("BootstrapState is nil, call ClassifyBootstrapAction() first")
+	}
+
 	role := strings.TrimSpace(nctx.Config.Spec.ClusterRole)
 	wantVersion := normalizeKubeVersion(strings.TrimSpace(nctx.Config.Spec.KubernetesVersion))
 	if wantVersion == "" {
@@ -122,7 +127,7 @@ func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error {
 
 	switch nctx.LocalClusterState.MembershipKind {
 	case LocalMembershipFresh:
-		// Fresh node has no existing cluster membership to compare against.
+		// Nothing to compare for fresh nodes.
 		return nil
 	case LocalMembershipPartial:
 		return fmt.Errorf("cannot check version skew with partial local cluster state")
@@ -142,6 +147,10 @@ func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error {
 		// Worker path stays permissive.
 		nctx.BootstrapState.UnsupportedWorkerVersionSkew = true
 		nctx.BootstrapState.VersionSkewReason = "cluster version could not be determined"
+
+		if nctx.BootstrapState.Action == BootstrapActionManageWorker {
+			nctx.BootstrapState.Action = BootstrapActionReconcileWorker
+		}
 		return nil
 	}
 
@@ -156,6 +165,14 @@ func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error {
 			)
 		}
 
+		if nctx.BootstrapState.Action == BootstrapActionManageControlPlane {
+			if versionEq(currentVersion, wantVersion) {
+				nctx.BootstrapState.Action = BootstrapActionReconcileControlPlane
+			} else {
+				nctx.BootstrapState.Action = BootstrapActionUpgradeControlPlane
+			}
+		}
+
 	case "worker":
 		if !isSupportedWorkerSkew(currentVersion, wantVersion) {
 			nctx.BootstrapState.UnsupportedWorkerVersionSkew = true
@@ -165,6 +182,14 @@ func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error {
 			)
 		}
 
+		if nctx.BootstrapState.Action == BootstrapActionManageWorker {
+			if versionEq(currentVersion, wantVersion) {
+				nctx.BootstrapState.Action = BootstrapActionReconcileWorker
+			} else {
+				nctx.BootstrapState.Action = BootstrapActionUpgradeWorker
+			}
+		}
+
 	default:
 		return fmt.Errorf("unsupported cluster role %q", role)
 	}
@@ -187,10 +212,8 @@ func ClassifyBootstrapAction(ctx context.Context, nctx *NodeContext) error {
 	}
 
 	state := &BootstrapState{}
-
-	// Preserve already-detected info if earlier steps populated it.
 	if nctx.BootstrapState != nil {
-		state.DetectedClusterVersion = nctx.BootstrapState.DetectedClusterVersion
+		*state = *nctx.BootstrapState
 	}
 
 	switch role {
@@ -200,13 +223,7 @@ func ClassifyBootstrapAction(ctx context.Context, nctx *NodeContext) error {
 			state.Action = BootstrapActionJoinWorker
 
 		case LocalMembershipExistingWorker:
-			if state.DetectedClusterVersion == "" {
+			state.Action = BootstrapActionManageWorker
-				state.Action = BootstrapActionReconcileWorker
-			} else if versionEq(state.DetectedClusterVersion, wantVersion) {
-				state.Action = BootstrapActionReconcileWorker
-			} else {
-				state.Action = BootstrapActionUpgradeWorker
-			}
 
 		case LocalMembershipExistingControlPlane, LocalMembershipPartial:
 			return fmt.Errorf("local state %q is invalid for worker role", nctx.LocalClusterState.MembershipKind)
@@ -225,15 +242,7 @@ func ClassifyBootstrapAction(ctx context.Context, nctx *NodeContext) error {
 			}
 
 		case LocalMembershipExistingControlPlane:
-			if state.DetectedClusterVersion == "" {
+			state.Action = BootstrapActionManageControlPlane
-				return errors.New("existing control-plane state found, but detected cluster version is empty")
-			}
-
-			if versionEq(state.DetectedClusterVersion, wantVersion) {
-				state.Action = BootstrapActionReconcileControlPlane
-			} else {
-				state.Action = BootstrapActionUpgradeControlPlane
-			}
 
 		case LocalMembershipExistingWorker:
 			return fmt.Errorf("local state %q is invalid for control-plane role", nctx.LocalClusterState.MembershipKind)
@@ -250,7 +259,7 @@ func ClassifyBootstrapAction(ctx context.Context, nctx *NodeContext) error {
 	}
 
 	nctx.BootstrapState = state
-	klog.V(4).Infof("Bootstrap action: %+v", *state)
+	klog.V(4).Infof("Bootstrap action classified: %+v", *state)
 	return nil
 }
 
@@ -521,7 +530,10 @@ func checkImagePresent(ctx context.Context, n *NodeContext, image string) error
 	}
 
 	// crictl inspecti exits non-zero when the image is absent.
-	_, err := n.SystemRunner.Run(ctx, "crictl", "inspecti", image)
+	_, err := n.SystemRunner.RunRetry(ctx, system.RetryOptions{
+		Attempts: 3,
+		Delay:    1 * system.DefaultSecond,
+	}, "crictl", "inspecti", image)
 	if err != nil {
 		return fmt.Errorf("image %q not present: %w", image, err)
 	}
@@ -553,7 +565,43 @@ func normalizeKubeVersion(v string) string {
 	return v
 }
 
-func GenerateKubeadmConfig(_ context.Context, nctx *NodeContext) error {
+func buildNodeRegistration(spec types.MonoKSConfigSpec) NodeRegistrationOptions {
+	nodeName := strings.TrimSpace(spec.NodeName)
+	criSocket := strings.TrimSpace(spec.ContainerRuntimeEndpoint)
+	advertiseAddress := strings.TrimSpace(spec.APIServerAdvertiseAddress)
+
+	nr := NodeRegistrationOptions{
+		Name:            nodeName,
+		CRISocket:       criSocket,
+		ImagePullPolicy: "IfNotPresent",
+		KubeletExtraArgs: []KubeadmArg{
+			{Name: "hostname-override", Value: nodeName},
+			{Name: "pod-manifest-path", Value: "/etc/kubernetes/manifests"},
+		},
+	}
+
+	if advertiseAddress != "" {
+		nr.KubeletExtraArgs = append(nr.KubeletExtraArgs,
+			KubeadmArg{Name: "node-ip", Value: advertiseAddress},
+		)
+	}
+
+	return nr
+}
+
+func maybeAddBootstrapTaint(nr *NodeRegistrationOptions, role string) {
+	if strings.TrimSpace(role) != "worker" {
+		return
+	}
+	nr.Taints = []KubeadmTaint{
+		{
+			Key:    "monok8s/bootstrap",
+			Effect: "NoSchedule",
+		},
+	}
+}
+
+func GenerateKubeadmInitConfig(_ context.Context, nctx *NodeContext) error {
 	if nctx == nil {
 		return fmt.Errorf("node context is nil")
 	}
@@ -564,47 +612,29 @@ func GenerateKubeadmConfig(_ context.Context, nctx *NodeContext) error {
 	if advertiseAddress == "" {
 		return fmt.Errorf("api server advertise address is required")
 	}
-
-	nodeName := strings.TrimSpace(spec.NodeName)
-	if nodeName == "" {
-		return fmt.Errorf("node name is required")
-	}
-
-	criSocket := strings.TrimSpace(spec.ContainerRuntimeEndpoint)
-	if criSocket == "" {
-		return fmt.Errorf("container runtime endpoint is required")
-	}
-
 	clusterName := strings.TrimSpace(spec.ClusterName)
 	if clusterName == "" {
 		return fmt.Errorf("cluster name is required")
 	}
-
 	kubernetesVersion := strings.TrimSpace(spec.KubernetesVersion)
 	if kubernetesVersion == "" {
 		return fmt.Errorf("kubernetes version is required")
 	}
-
 	podSubnet := strings.TrimSpace(spec.PodSubnet)
 	if podSubnet == "" {
 		return fmt.Errorf("pod subnet is required")
 	}
-
 	serviceSubnet := strings.TrimSpace(spec.ServiceSubnet)
 	if serviceSubnet == "" {
 		return fmt.Errorf("service subnet is required")
 	}
-
 	clusterDomain := strings.TrimSpace(spec.ClusterDomain)
 	if clusterDomain == "" {
 		return fmt.Errorf("cluster domain is required")
 	}
 
 	certSANs := []string{advertiseAddress}
-	seen := map[string]struct{}{
+	seen := map[string]struct{}{advertiseAddress: {}}
-		advertiseAddress: {},
-	}
-
 	for _, raw := range spec.SubjectAltNames {
 		san := strings.TrimSpace(raw)
 		if san == "" {
@@ -617,94 +647,129 @@ func GenerateKubeadmConfig(_ context.Context, nctx *NodeContext) error {
 		certSANs = append(certSANs, san)
 	}
 
-	type kubeadmInitConfiguration struct {
+	nodeReg := buildNodeRegistration(spec)
-		APIVersion       string `yaml:"apiVersion"`
+	if spec.ClusterRole == "worker" {
-		Kind             string `yaml:"kind"`
+		maybeAddBootstrapTaint(&nodeReg, spec.ClusterRole)
-		LocalAPIEndpoint struct {
-			AdvertiseAddress string `yaml:"advertiseAddress"`
-			BindPort         int    `yaml:"bindPort"`
-		} `yaml:"localAPIEndpoint"`
-		NodeRegistration struct {
-			Name             string `yaml:"name"`
-			CRISocket        string `yaml:"criSocket"`
-			ImagePullPolicy  string `yaml:"imagePullPolicy"`
-			KubeletExtraArgs []struct {
-				Name  string `yaml:"name"`
-				Value string `yaml:"value"`
-			} `yaml:"kubeletExtraArgs"`
-		} `yaml:"nodeRegistration"`
 	}
 
-	type kubeadmClusterConfiguration struct {
+	initCfg := InitConfiguration{
-		APIVersion        string `yaml:"apiVersion"`
-		Kind              string `yaml:"kind"`
-		ClusterName       string `yaml:"clusterName"`
-		KubernetesVersion string `yaml:"kubernetesVersion"`
-		Networking        struct {
-			PodSubnet     string `yaml:"podSubnet"`
-			ServiceSubnet string `yaml:"serviceSubnet"`
-			DNSDomain     string `yaml:"dnsDomain"`
-		} `yaml:"networking"`
-		APIServer struct {
-			CertSANs []string `yaml:"certSANs"`
-		} `yaml:"apiServer"`
-	}
-
-	type kubeletConfiguration struct {
-		APIVersion               string `yaml:"apiVersion"`
-		Kind                     string `yaml:"kind"`
-		CgroupDriver             string `yaml:"cgroupDriver"`
-		ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint"`
-	}
-
-	initCfg := kubeadmInitConfiguration{
 		APIVersion: "kubeadm.k8s.io/v1beta4",
 		Kind:       "InitConfiguration",
-	}
+		LocalAPIEndpoint: LocalAPIEndpoint{
-	initCfg.LocalAPIEndpoint.AdvertiseAddress = advertiseAddress
+			AdvertiseAddress: advertiseAddress,
-	initCfg.LocalAPIEndpoint.BindPort = 6443
+			BindPort:         6443,
-	initCfg.NodeRegistration.Name = nodeName
+		},
-	initCfg.NodeRegistration.CRISocket = criSocket
+		NodeRegistration: nodeReg,
-	initCfg.NodeRegistration.ImagePullPolicy = "IfNotPresent"
-	initCfg.NodeRegistration.KubeletExtraArgs = []struct {
-		Name  string `yaml:"name"`
-		Value string `yaml:"value"`
-	}{
-		{Name: "hostname-override", Value: nodeName},
-		{Name: "node-ip", Value: advertiseAddress},
-		{Name: "pod-manifest-path", Value: "/etc/kubernetes/manifests"},
 	}
 
-	clusterCfg := kubeadmClusterConfiguration{
+	clusterCfg := ClusterConfiguration{
 		APIVersion:        "kubeadm.k8s.io/v1beta4",
 		Kind:              "ClusterConfiguration",
 		ClusterName:       clusterName,
 		KubernetesVersion: kubernetesVersion,
+		Networking: Networking{
+			PodSubnet:     podSubnet,
+			ServiceSubnet: serviceSubnet,
+			DNSDomain:     clusterDomain,
+		},
+		APIServer: APIServer{
+			CertSANs: certSANs,
+		},
 	}
-	clusterCfg.Networking.PodSubnet = podSubnet
-	clusterCfg.Networking.ServiceSubnet = serviceSubnet
-	clusterCfg.Networking.DNSDomain = clusterDomain
-	clusterCfg.APIServer.CertSANs = certSANs
 
-	kubeletCfg := kubeletConfiguration{
+	kubeletCfg := KubeletConfiguration{
 		APIVersion:               "kubelet.config.k8s.io/v1beta1",
 		Kind:                     "KubeletConfiguration",
 		CgroupDriver:             "cgroupfs",
-		ContainerRuntimeEndpoint: criSocket,
+		ContainerRuntimeEndpoint: strings.TrimSpace(spec.ContainerRuntimeEndpoint),
 	}
 
-	var docs [][]byte
+	return writeKubeadmYAML(tmpKubeadmInitConf, initCfg, clusterCfg, kubeletCfg)
+}
 
-	for _, doc := range []any{initCfg, clusterCfg, kubeletCfg} {
+func GenerateKubeadmJoinConfig(_ context.Context, nctx *NodeContext) error {
+	if nctx == nil {
+		return fmt.Errorf("node context is nil")
+	}
+
+	spec := nctx.Config.Spec
+
+	apiServerEndpoint := strings.TrimSpace(spec.APIServerEndpoint)
+	if apiServerEndpoint == "" {
+		return fmt.Errorf("spec.apiServerEndpoint is required")
+	}
+
+	bootstrapToken := strings.TrimSpace(spec.BootstrapToken)
+	if bootstrapToken == "" {
+		return fmt.Errorf("spec.bootstrapToken is required")
+	}
+
+	discoveryTokenCACertHash := strings.TrimSpace(spec.DiscoveryTokenCACertHash)
+	if discoveryTokenCACertHash == "" {
+		return fmt.Errorf("spec.discoveryTokenCACertHash is required")
+	}
+
+	nodeReg := buildNodeRegistration(spec)
+	if nctx.BootstrapState != nil && nctx.BootstrapState.Action == BootstrapActionJoinWorker {
+		maybeAddBootstrapTaint(&nodeReg, spec.ClusterRole)
+	}
+
+	joinCfg := JoinConfiguration{
+		APIVersion:       "kubeadm.k8s.io/v1beta4",
+		Kind:             "JoinConfiguration",
+		NodeRegistration: nodeReg,
+		Discovery: Discovery{
+			BootstrapToken: BootstrapTokenDiscovery{
+				APIServerEndpoint: apiServerEndpoint,
+				Token:             bootstrapToken,
+				CACertHashes:      []string{discoveryTokenCACertHash},
+			},
+		},
+	}
+
+	if nctx.BootstrapState != nil && nctx.BootstrapState.Action == BootstrapActionJoinControlPlane {
+		certKey := strings.TrimSpace(spec.ControlPlaneCertKey)
+		if certKey == "" {
+			return fmt.Errorf("spec.controlPlaneCertKey is required for control-plane join")
+		}
+
+		advertiseAddress := strings.TrimSpace(spec.APIServerAdvertiseAddress)
+		if advertiseAddress == "" {
+			return fmt.Errorf("spec.apiServerAdvertiseAddress is required for control-plane join")
+		}
+
+		joinCfg.ControlPlane = &JoinControlPlane{
+			CertificateKey: certKey,
+		}
+		joinCfg.LocalAPIEndpoint = &LocalAPIEndpoint{
+			AdvertiseAddress: advertiseAddress,
+			BindPort:         6443,
+		}
+	}
+
+	kubeletCfg := KubeletConfiguration{
+		APIVersion:               "kubelet.config.k8s.io/v1beta1",
+		Kind:                     "KubeletConfiguration",
+		CgroupDriver:             "cgroupfs",
+		ContainerRuntimeEndpoint: strings.TrimSpace(spec.ContainerRuntimeEndpoint),
+	}
+
+	return writeKubeadmYAML(tmpKubeadmInitConf, joinCfg, kubeletCfg)
+}
+
+func writeKubeadmYAML(path string, docs ...any) error {
+	var renderedDocs [][]byte
+
+	for _, doc := range docs {
 		b, err := yaml.Marshal(doc)
 		if err != nil {
 			return fmt.Errorf("marshal kubeadm config document: %w", err)
 		}
-		docs = append(docs, bytes.TrimSpace(b))
+		renderedDocs = append(renderedDocs, bytes.TrimSpace(b))
 	}
 
 	var buf bytes.Buffer
-	for i, doc := range docs {
+	for i, doc := range renderedDocs {
 		if i > 0 {
 			buf.WriteString("\n---\n")
 		}
@@ -714,16 +779,25 @@ func GenerateKubeadmConfig(_ context.Context, nctx *NodeContext) error {
 
 	rendered := buf.String()
 
-	if err := os.WriteFile(tmpKubeadmInitConf, []byte(rendered), 0o600); err != nil {
+	if err := os.WriteFile(path, []byte(rendered), 0o600); err != nil {
-		return fmt.Errorf("write kubeadm config to %s: %w", tmpKubeadmInitConf, err)
+		return fmt.Errorf("write kubeadm config to %s: %w", path, err)
 	}
 
-	klog.V(4).Infof("generated kubeadm config at %s:\n%s", tmpKubeadmInitConf, rendered)
+	klog.V(4).Infof("generated kubeadm config at %s:\n%s", path, rendered)
 	return nil
 }
 
 func RunKubeadmInit(ctx context.Context, nctx *NodeContext) error {
-	if err := GenerateKubeadmConfig(ctx, nctx); err != nil {
+	if nctx.BootstrapState == nil {
+		return errors.New("BootstrapState is nil. Please run earlier steps first")
+	}
+
+	if nctx.BootstrapState.Action != BootstrapActionInitControlPlane {
+		klog.V(4).Infof("skipped for %s", nctx.BootstrapState.Action)
+		return nil
+	}
+
+	if err := GenerateKubeadmInitConfig(ctx, nctx); err != nil {
 		return err
 	}
 
@@ -740,39 +814,51 @@ func RunKubeadmInit(ctx context.Context, nctx *NodeContext) error {
 	return err
 }
 
+func RunKubeadmJoin(ctx context.Context, nctx *NodeContext) error {
+	if nctx.BootstrapState == nil {
+		return errors.New("BootstrapState is nil. Please run earlier steps first")
+	}
+
+	switch nctx.BootstrapState.Action {
+	case BootstrapActionJoinWorker, BootstrapActionJoinControlPlane:
+		// continue
+	default:
+		klog.V(4).Infof("RunKubeadmJoin skipped for action %q", nctx.BootstrapState.Action)
+		return nil
+	}
+
+	if err := GenerateKubeadmJoinConfig(ctx, nctx); err != nil {
+		return err
+	}
+
+	klog.Infof("running kubeadm join for action %q", nctx.BootstrapState.Action)
+
+	_, err := nctx.SystemRunner.RunWithOptions(
+		ctx,
+		"kubeadm",
+		[]string{"join", "--config", tmpKubeadmInitConf},
+		system.RunOptions{
+			Timeout: 5 * time.Minute,
+			OnStdoutLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+			OnStderrLine: func(line string) {
+				klog.Infof("[kubeadm] %s", line)
+			},
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("run kubeadm join: %w", err)
+	}
+
+	return nil
+}
+
 func RunKubeadmUpgradeApply(context.Context, *NodeContext) error {
 	klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply")
 	return nil
 }
 
-func RunKubeadmJoin(context.Context, *NodeContext) error {
-	/*
-		run_kubeadm_join() {
-		    log "running kubeadm join..."
-
-		    case "$JOIN_KIND" in
-		        worker)
-		            kubeadm join "${API_SERVER_ENDPOINT}" \
-		                --token "${BOOTSTRAP_TOKEN}" \
-		                --discovery-token-ca-cert-hash "${DISCOVERY_TOKEN_CA_CERT_HASH}" \
-		                --node-name "${NODE_NAME}" \
-		                --cri-socket "${CONTAINER_RUNTIME_ENDPOINT}"
-		            ;;
-		        control-plane)
-		            kubeadm join "${API_SERVER_ENDPOINT}" \
-		                --token "${BOOTSTRAP_TOKEN}" \
-		                --discovery-token-ca-cert-hash "${DISCOVERY_TOKEN_CA_CERT_HASH}" \
-		                --control-plane \
-		                --certificate-key "${CONTROL_PLANE_CERT_KEY}" \
-		                --apiserver-advertise-address "${APISERVER_ADVERTISE_ADDRESS}" \
-		                --node-name "${NODE_NAME}" \
-		                --cri-socket "${CONTAINER_RUNTIME_ENDPOINT}"
-		            ;;
-		    esac
-		}
-	*/
-	return nil
-}
 func RunKubeadmUpgradeNode(context.Context, *NodeContext) error {
 	klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node")
 	return nil

clitools/pkg/node/kubeadm_types.go

@@ -0,0 +1,81 @@
+package node
+
+type KubeadmArg struct {
+	Name  string `yaml:"name"`
+	Value string `yaml:"value"`
+}
+
+type KubeadmTaint struct {
+	Key    string `yaml:"key"`
+	Value  string `yaml:"value,omitempty"`
+	Effect string `yaml:"effect"`
+}
+
+type LocalAPIEndpoint struct {
+	AdvertiseAddress string `yaml:"advertiseAddress"`
+	BindPort         int    `yaml:"bindPort"`
+}
+
+type NodeRegistrationOptions struct {
+	Name             string         `yaml:"name"`
+	CRISocket        string         `yaml:"criSocket"`
+	ImagePullPolicy  string         `yaml:"imagePullPolicy,omitempty"`
+	Taints           []KubeadmTaint `yaml:"taints,omitempty"`
+	KubeletExtraArgs []KubeadmArg   `yaml:"kubeletExtraArgs,omitempty"`
+}
+
+type Networking struct {
+	PodSubnet     string `yaml:"podSubnet,omitempty"`
+	ServiceSubnet string `yaml:"serviceSubnet,omitempty"`
+	DNSDomain     string `yaml:"dnsDomain,omitempty"`
+}
+
+type APIServer struct {
+	CertSANs []string `yaml:"certSANs,omitempty"`
+}
+
+type InitConfiguration struct {
+	APIVersion       string                  `yaml:"apiVersion"`
+	Kind             string                  `yaml:"kind"`
+	LocalAPIEndpoint LocalAPIEndpoint        `yaml:"localAPIEndpoint"`
+	NodeRegistration NodeRegistrationOptions `yaml:"nodeRegistration"`
+}
+
+type ClusterConfiguration struct {
+	APIVersion        string     `yaml:"apiVersion"`
+	Kind              string     `yaml:"kind"`
+	ClusterName       string     `yaml:"clusterName"`
+	KubernetesVersion string     `yaml:"kubernetesVersion"`
+	Networking        Networking `yaml:"networking"`
+	APIServer         APIServer  `yaml:"apiServer"`
+}
+
+type BootstrapTokenDiscovery struct {
+	APIServerEndpoint string   `yaml:"apiServerEndpoint"`
+	Token             string   `yaml:"token"`
+	CACertHashes      []string `yaml:"caCertHashes"`
+}
+
+type Discovery struct {
+	BootstrapToken BootstrapTokenDiscovery `yaml:"bootstrapToken"`
+}
+
+type JoinControlPlane struct {
+	CertificateKey string `yaml:"certificateKey"`
+}
+
+type JoinConfiguration struct {
+	APIVersion       string                  `yaml:"apiVersion"`
+	Kind             string                  `yaml:"kind"`
+	NodeRegistration NodeRegistrationOptions `yaml:"nodeRegistration"`
+	Discovery        Discovery               `yaml:"discovery"`
+	ControlPlane     *JoinControlPlane       `yaml:"controlPlane,omitempty"`
+	LocalAPIEndpoint *LocalAPIEndpoint       `yaml:"localAPIEndpoint,omitempty"`
+}
+
+type KubeletConfiguration struct {
+	APIVersion               string `yaml:"apiVersion"`
+	Kind                     string `yaml:"kind"`
+	CgroupDriver             string `yaml:"cgroupDriver"`
+	ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint"`
+}

clitools/pkg/templates/templates.go

@@ -8,6 +8,7 @@ import (
 	buildinfo "undecided.project/monok8s/pkg/buildinfo"
 )
 
+var ValAPIServerEndPoint string = "10.0.0.10:6443"
 var ValHostname string = "monoks-master-1"
 var ValBootstrapToken string = "abcd12.ef3456789abcdef0"
 var ValDiscoveryTokenCACertHash string = "sha256:9f1c2b3a4d5e6f7890abc1234567890abcdef1234567890abcdef1234567890ab"
@@ -16,6 +17,7 @@ func init() {
 	ValBootstrapToken = os.Getenv("HOSTNAME")
 	ValBootstrapToken = os.Getenv("BOOTSTRAP_TOKEN")
 	ValDiscoveryTokenCACertHash = os.Getenv("TOKEN_CACERT_HASH")
+	ValAPIServerEndPoint = os.Getenv("API_SERVER_ENDPOINT")
 }
 
 func DefaultMonoKSConfig() types.MonoKSConfig {
@@ -42,7 +44,7 @@ func DefaultMonoKSConfig() types.MonoKSConfig {
 			ServiceSubnet: "10.96.0.0/12",
 
 			APIServerAdvertiseAddress: "10.0.0.10",
-			APIServerEndpoint:         "10.0.0.10:6443",
+			APIServerEndpoint:         ValAPIServerEndPoint,
 
 			// Fake token and hash for placeholder purpose
 			BootstrapToken:           ValBootstrapToken,