Refine controller template and probe listeners

clitools/pkg/cmd/agent/agent.go

@@ -152,14 +152,11 @@ func watchOnce(
 		if !targetsNode(item, nodeName) {
 			continue
 		}
-		if !shouldHandle(item) {
-			continue
-		}
 
 		klog.InfoS("found existing osupgradeprogress",
 			"name", item.Name,
 			"node", nodeName,
-			"phase", progressPhase(item.Status),
+			"phase", item.StatusPhase(),
 			"resourceVersion", item.ResourceVersion,
 		)
 
@@ -227,20 +224,11 @@ func watchOnce(
 			if !targetsNode(osup, nodeName) {
 				continue
 			}
-			if !shouldHandle(osup) {
-				klog.V(2).InfoS("skipping osupgradeprogress due to phase",
-					"name", osup.Name,
-					"node", nodeName,
-					"phase", progressPhase(osup.Status),
-					"eventType", evt.Type,
-				)
-				continue
-			}
 
-			klog.InfoS("received osupgradeprogress event",
+			klog.V(4).InfoS("received osupgradeprogress event",
 				"name", osup.Name,
 				"node", nodeName,
-				"phase", progressPhase(osup.Status),
+				"phase", osup.StatusPhase(),
 				"eventType", evt.Type,
 				"resourceVersion", osup.ResourceVersion,
 			)
@@ -262,28 +250,3 @@ func targetsNode(osup *monov1alpha1.OSUpgradeProgress, nodeName string) bool {
 	}
 	return osup.Spec.NodeName == nodeName
 }
-
-func shouldHandle(osup *monov1alpha1.OSUpgradeProgress) bool {
-	if osup == nil {
-		return false
-	}
-
-	if osup.Status == nil {
-		return false
-	}
-
-	switch osup.Status.Phase {
-	case "",
-		monov1alpha1.OSUpgradeProgressPhasePending:
-		return true
-	default:
-		return false
-	}
-}
-
-func progressPhase(st *monov1alpha1.OSUpgradeProgressStatus) string {
-	if st == nil {
-		return ""
-	}
-	return string(st.Phase)
-}

clitools/pkg/cmd/controller/controller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -57,7 +58,7 @@ func NewCmdController(flags *genericclioptions.ConfigFlags) *cobra.Command {
 			}()
 
 			go func() {
-				httpErrCh <- httpListen(ctx, clients, conf)
+				httpErrCh <- listenAndServe(ctx, clients, conf)
 			}()
 
 			select {
@@ -92,63 +93,125 @@ func NewCmdController(flags *genericclioptions.ConfigFlags) *cobra.Command {
 	return cmd
 }
 
-func httpListen(ctx context.Context, clients *kube.Clients, conf ServerConfig) error {
-	address, port := "", "8443"
-	addr := net.JoinHostPort(address, port)
-
+func listenAndServe(ctx context.Context, clients *kube.Clients, conf ServerConfig) error {
 	nodeName := os.Getenv("NODE_NAME")
 
-	server := mkscontroller.NewServer(ctx, clients, conf.Namespace, nodeName)
+	controllerServer := mkscontroller.NewServer(ctx, clients, conf.Namespace, nodeName)
 
-	s := &http.Server{
-		Addr:           addr,
-		Handler:        server,
+	healthMux := http.NewServeMux()
+	healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+	healthMux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+
+	healthAddr := net.JoinHostPort("", "8080")
+	controllerAddr := net.JoinHostPort("", "8443")
+
+	healthHTTPServer := &http.Server{
+		Addr:           healthAddr,
+		Handler:        healthMux,
+		IdleTimeout:    90 * time.Second,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	controllerHTTPServer := &http.Server{
+		Addr:           controllerAddr,
+		Handler:        controllerServer,
 		IdleTimeout:    90 * time.Second,
 		ReadTimeout:    4 * time.Minute,
 		WriteTimeout:   4 * time.Minute,
 		MaxHeaderBytes: 1 << 20,
 	}
 
-	serverErrCh := make(chan error, 1)
+	serverErrCh := make(chan error, 2)
 
 	go func() {
-		if conf.TLSCertFile != "" {
-			klog.InfoS("starting HTTPS server",
-				"addr", addr,
-				"certFile", conf.TLSCertFile,
-				"keyFile", conf.TLSPrivateKeyFile,
-			)
-			serverErrCh <- s.ListenAndServeTLS(conf.TLSCertFile, conf.TLSPrivateKeyFile)
+		klog.InfoS("starting health HTTP server", "addr", healthAddr)
+
+		err := healthHTTPServer.ListenAndServe()
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			serverErrCh <- fmt.Errorf("health HTTP server: %w", err)
 			return
 		}
 
-		klog.InfoS("starting HTTP server", "addr", addr)
-		serverErrCh <- s.ListenAndServe()
+		serverErrCh <- nil
+	}()
+
+	go func() {
+		if conf.TLSCertFile != "" {
+			klog.InfoS("starting controller HTTPS server",
+				"addr", controllerAddr,
+				"certFile", conf.TLSCertFile,
+				"keyFile", conf.TLSPrivateKeyFile,
+			)
+
+			err := controllerHTTPServer.ListenAndServeTLS(conf.TLSCertFile, conf.TLSPrivateKeyFile)
+			if err != nil && !errors.Is(err, http.ErrServerClosed) {
+				serverErrCh <- fmt.Errorf("controller HTTPS server: %w", err)
+				return
+			}
+
+			serverErrCh <- nil
+			return
+		}
+
+		klog.InfoS("starting controller HTTP server", "addr", controllerAddr)
+
+		err := controllerHTTPServer.ListenAndServe()
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			serverErrCh <- fmt.Errorf("controller HTTP server: %w", err)
+			return
+		}
+
+		serverErrCh <- nil
 	}()
 
 	select {
 	case <-ctx.Done():
-		klog.InfoS("shutting down HTTP server", "addr", addr)
+		klog.InfoS("shutting down HTTP servers",
+			"healthAddr", healthAddr,
+			"controllerAddr", controllerAddr,
+		)
 
 		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
 
-		err := s.Shutdown(shutdownCtx)
-		if err != nil {
-			return err
+		var errs []error
+
+		if err := healthHTTPServer.Shutdown(shutdownCtx); err != nil {
+			errs = append(errs, fmt.Errorf("shutdown health HTTP server: %w", err))
 		}
 
-		err = <-serverErrCh
-		if err != nil && !errors.Is(err, http.ErrServerClosed) {
-			return err
+		if err := controllerHTTPServer.Shutdown(shutdownCtx); err != nil {
+			errs = append(errs, fmt.Errorf("shutdown controller HTTP server: %w", err))
 		}
+
+		for i := 0; i < 2; i++ {
+			if err := <-serverErrCh; err != nil {
+				errs = append(errs, err)
+			}
+		}
+
+		if len(errs) > 0 {
+			return errors.Join(errs...)
+		}
+
 		return context.Canceled
 
 	case err := <-serverErrCh:
-		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+		if err != nil {
 			klog.ErrorS(err, "HTTP server failed")
 			return err
 		}
-		return nil
+
+		// One server exited cleanly unexpectedly. Treat that as failure because
+		// the process should keep both servers alive until ctx is canceled.
+		return fmt.Errorf("HTTP server exited unexpectedly")
 	}
 }

clitools/pkg/cmd/create/create.go

@@ -42,23 +42,6 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command {
 				return err
 			},
 		},
-		&cobra.Command{
-			Use:   "controller",
-			Short: "Print controller deployment template",
-			RunE: func(cmd *cobra.Command, _ []string) error {
-				ns, _, err := flags.ToRawKubeConfigLoader().Namespace()
-				if err != nil {
-					return err
-				}
-
-				out, err := render.RenderControllerDeployments(ns)
-				if err != nil {
-					return err
-				}
-				_, err = fmt.Fprint(cmd.OutOrStdout(), out)
-				return err
-			},
-		},
 	)
 
 	var authorizedKeysPath string
@@ -90,6 +73,38 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command {
 	sshdcmd.Flags().StringVar(&authorizedKeysPath, "authkeys", "", "path to authorized_keys file")
 
 	cmd.AddCommand(&sshdcmd)
+
+	cconf := render.ControllerConf{}
+	controllercmd := cobra.Command{
+		Use:   "controller",
+		Short: "Print controller deployment template",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			ns, _, err := flags.ToRawKubeConfigLoader().Namespace()
+			if err != nil {
+				return err
+			}
+
+			cconf.Namespace = ns
+
+			out, err := render.RenderControllerDeployments(cconf)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprint(cmd.OutOrStdout(), out)
+			return err
+		},
+	}
+
+	controllercmd.Flags().StringVar(
+		&cconf.Image,
+		"image",
+		"",
+		"Controller image, including optional registry and tag",
+	)
+
+	cmd.AddCommand(&controllercmd)
+
 	return cmd
 }
 

clitools/pkg/cmd/root/root.go

@@ -19,8 +19,10 @@ import (
 func init() {
 	klog.InitFlags(nil)
 
+	_ = flag.Set("logtostderr", "true")
+
 	if os.Getenv("DEBUG") != "" {
-		_ = flag.Set("v", "4") // debug level
+		_ = flag.Set("v", "4")
 	} else {
 		_ = flag.Set("v", "0")
 	}
@@ -39,7 +41,11 @@ func NewRootCmd() *cobra.Command {
 		},
 	}
 
+	// Expose klog stdlib flags through Cobra/pflag.
+	cmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
+
 	flags.AddFlags(cmd.PersistentFlags())
+
 	cmd.AddCommand(
 		versioncmd.NewCmdVersion(),
 		initcmd.NewCmdInit(flags),
@@ -49,5 +55,6 @@ func NewRootCmd() *cobra.Command {
 		controllercmd.NewCmdController(flags),
 		internalcmd.NewCmdInternal(),
 	)
+
 	return cmd
 }