Refine controller template and probe listeners

clitools/pkg/render/controller.go

@@ -14,13 +14,20 @@ import (
 
 	monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
 	buildinfo "example.com/monok8s/pkg/buildinfo"
-	templates "example.com/monok8s/pkg/templates"
 )
 
-func RenderControllerDeployments(namespace string) (string, error) {
-	vals := templates.LoadTemplateValuesFromEnv()
+type ControllerConf struct {
+	Namespace string
+	Image     string
+	Labels    map[string]string
+}
 
-	labels := map[string]string{
+func RenderControllerDeployments(conf ControllerConf) (string, error) {
+	if conf.Namespace == "" {
+		return "", fmt.Errorf("namespace is required")
+	}
+
+	conf.Labels = map[string]string{
 		"app.kubernetes.io/name":       monov1alpha1.ControllerName,
 		"app.kubernetes.io/component":  "controller",
 		"app.kubernetes.io/part-of":    "monok8s",
@@ -28,10 +35,10 @@ func RenderControllerDeployments(namespace string) (string, error) {
 	}
 
 	objs := []runtime.Object{
-		buildControllerServiceAccount(namespace, labels),
-		buildControllerClusterRole(labels),
-		buildControllerClusterRoleBinding(namespace, labels),
-		buildControllerDeployment(vals, namespace, labels),
+		buildControllerServiceAccount(conf),
+		buildControllerClusterRole(conf),
+		buildControllerClusterRoleBinding(conf),
+		buildControllerDeployment(conf),
 	}
 
 	s := runtime.NewScheme()
@@ -57,7 +64,7 @@ func RenderControllerDeployments(namespace string) (string, error) {
 	return buf.String(), nil
 }
 
-func buildControllerServiceAccount(namespace string, labels map[string]string) *corev1.ServiceAccount {
+func buildControllerServiceAccount(conf ControllerConf) *corev1.ServiceAccount {
 
 	automount := true
 
@@ -68,14 +75,14 @@ func buildControllerServiceAccount(namespace string, labels map[string]string) *
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      monov1alpha1.ControllerName,
-			Namespace: namespace,
-			Labels:    labels,
+			Namespace: conf.Namespace,
+			Labels:    conf.Labels,
 		},
 		AutomountServiceAccountToken: &automount,
 	}
 }
 
-func buildControllerClusterRole(labels map[string]string) *rbacv1.ClusterRole {
+func buildControllerClusterRole(conf ControllerConf) *rbacv1.ClusterRole {
 	wantRules := []rbacv1.PolicyRule{
 		{
 			APIGroups: []string{monov1alpha1.Group},
@@ -111,19 +118,19 @@ func buildControllerClusterRole(labels map[string]string) *rbacv1.ClusterRole {
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   monov1alpha1.ControllerName,
-			Labels: labels,
+			Labels: conf.Labels,
 		},
 		Rules: wantRules,
 	}
 }
 
-func buildControllerClusterRoleBinding(namespace string, labels map[string]string) *rbacv1.ClusterRoleBinding {
+func buildControllerClusterRoleBinding(conf ControllerConf) *rbacv1.ClusterRoleBinding {
 
 	wantSubjects := []rbacv1.Subject{
 		{
 			Kind:      "ServiceAccount",
 			Name:      monov1alpha1.ControllerName,
-			Namespace: namespace,
+			Namespace: conf.Namespace,
 		},
 	}
 
@@ -140,14 +147,14 @@ func buildControllerClusterRoleBinding(namespace string, labels map[string]strin
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   monov1alpha1.ControllerName,
-			Labels: labels,
+			Labels: conf.Labels,
 		},
 		Subjects: wantSubjects,
 		RoleRef:  wantRoleRef,
 	}
 }
 
-func buildControllerDeployment(tVals templates.TemplateValues, namespace string, labels map[string]string) *appsv1.Deployment {
+func buildControllerDeployment(conf ControllerConf) *appsv1.Deployment {
 	replicas := int32(1)
 
 	selectorLabels := map[string]string{
@@ -155,10 +162,13 @@ func buildControllerDeployment(tVals templates.TemplateValues, namespace string,
 		"app.kubernetes.io/component": "controller",
 	}
 
-	podLabels := mergeStringMaps(labels, selectorLabels)
+	podLabels := mergeStringMaps(conf.Labels, selectorLabels)
 
 	runAsNonRoot := true
 	allowPrivilegeEscalation := false
+	userGroup := int64(65532)
+
+	image, pullPolicy := controllerImage(conf)
 
 	return &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
@@ -167,8 +177,8 @@ func buildControllerDeployment(tVals templates.TemplateValues, namespace string,
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      monov1alpha1.ControllerName,
-			Namespace: namespace,
-			Labels:    labels,
+			Namespace: conf.Namespace,
+			Labels:    conf.Labels,
 		},
 		Spec: appsv1.DeploymentSpec{
 			Replicas: &replicas,
@@ -184,12 +194,12 @@ func buildControllerDeployment(tVals templates.TemplateValues, namespace string,
 					Containers: []corev1.Container{
 						{
 							Name:            "controller",
-							Image:           fmt.Sprintf("localhost/monok8s/node-control:%s", buildinfo.Version),
-							ImagePullPolicy: corev1.PullIfNotPresent,
+							Image:           image,
+							ImagePullPolicy: pullPolicy,
 							Args: []string{
 								"controller",
 								"--namespace",
-								namespace,
+								conf.Namespace,
 							},
 							Env: []corev1.EnvVar{
 								{
@@ -239,6 +249,10 @@ func buildControllerDeployment(tVals templates.TemplateValues, namespace string,
 										Port: intstr.FromString("http"),
 									},
 								},
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       60,
+								TimeoutSeconds:      2,
+								FailureThreshold:    3,
 							},
 							ReadinessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
@@ -247,13 +261,64 @@ func buildControllerDeployment(tVals templates.TemplateValues, namespace string,
 										Port: intstr.FromString("http"),
 									},
 								},
+								InitialDelaySeconds: 2,
+								PeriodSeconds:       5,
+								TimeoutSeconds:      2,
+								FailureThreshold:    3,
 							},
 							SecurityContext: &corev1.SecurityContext{
 								RunAsNonRoot:             &runAsNonRoot,
+								RunAsUser:                &userGroup,
+								RunAsGroup:               &userGroup,
 								AllowPrivilegeEscalation: &allowPrivilegeEscalation,
 							},
 						},
 					},
+					NodeSelector: controllerNodeSelector(conf),
+					Affinity:     controllerAffinity(conf),
+				},
+			},
+		},
+	}
+}
+
+func controllerImage(conf ControllerConf) (string, corev1.PullPolicy) {
+	if conf.Image != "" {
+		return conf.Image, corev1.PullIfNotPresent
+	}
+
+	return fmt.Sprintf("localhost/monok8s/node-control:%s", buildinfo.Version), corev1.PullNever
+}
+
+func controllerNodeSelector(conf ControllerConf) map[string]string {
+	if conf.Image != "" {
+		return nil
+	}
+
+	// Local image exists on managed nodes only.
+	return map[string]string{
+		monov1alpha1.NodeControlKey: "true",
+	}
+}
+
+func controllerAffinity(conf ControllerConf) *corev1.Affinity {
+	// Local image exists only on managed nodes, so in that mode we already use
+	// NodeSelector and should not fight placement with anti-affinity.
+	if conf.Image == "" {
+		return nil
+	}
+
+	return &corev1.Affinity{
+		PodAntiAffinity: &corev1.PodAntiAffinity{
+			PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+				{
+					Weight: 100,
+					PodAffinityTerm: corev1.PodAffinityTerm{
+						TopologyKey: corev1.LabelHostname,
+						LabelSelector: &metav1.LabelSelector{
+							MatchLabels: monov1alpha1.NodeAgentLabels(),
+						},
+					},
 				},
 			},
 		},