package render import ( "bytes" "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/intstr" monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" buildinfo "example.com/monok8s/pkg/buildinfo" ) type ControllerConf struct { Namespace string Image string Labels map[string]string } func RenderControllerDeployments(conf ControllerConf) (string, error) { if conf.Namespace == "" { return "", fmt.Errorf("namespace is required") } conf.Labels = map[string]string{ "app.kubernetes.io/name": monov1alpha1.ControllerName, "app.kubernetes.io/component": "controller", "app.kubernetes.io/part-of": "monok8s", "app.kubernetes.io/managed-by": monov1alpha1.NodeControlName, } objs := []runtime.Object{ buildControllerServiceAccount(conf), buildControllerClusterRole(conf), buildControllerClusterRoleBinding(conf), buildControllerDeployment(conf), } s := runtime.NewScheme() _ = corev1.AddToScheme(s) _ = rbacv1.AddToScheme(s) _ = appsv1.AddToScheme(s) serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, s, s) var buf bytes.Buffer for i, obj := range objs { if i > 0 { if _, err := fmt.Fprintln(&buf, "---"); err != nil { return "", err } } if err := serializer.Encode(obj, &buf); err != nil { return "", err } } return buf.String(), nil } func buildControllerServiceAccount(conf ControllerConf) *corev1.ServiceAccount { automount := true return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ Name: monov1alpha1.ControllerName, Namespace: conf.Namespace, Labels: conf.Labels, }, AutomountServiceAccountToken: &automount, } } func buildControllerClusterRole(conf ControllerConf) *rbacv1.ClusterRole { wantRules := []rbacv1.PolicyRule{ { APIGroups: []string{monov1alpha1.Group}, Resources: []string{"osupgrades"}, Verbs: []string{"get", "list", "watch"}, }, { APIGroups: []string{monov1alpha1.Group}, Resources: []string{"osupgrades/status"}, Verbs: []string{"get", "patch", "update"}, }, { APIGroups: []string{monov1alpha1.Group}, Resources: []string{"osupgradeprogresses"}, Verbs: []string{"get", "create"}, }, { APIGroups: []string{monov1alpha1.Group}, Resources: []string{"osupgradeprogresses/status"}, Verbs: []string{"update"}, }, { APIGroups: []string{""}, Resources: []string{"nodes"}, Verbs: []string{"get", "list"}, }, } return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{ APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", }, ObjectMeta: metav1.ObjectMeta{ Name: monov1alpha1.ControllerName, Labels: conf.Labels, }, Rules: wantRules, } } func buildControllerClusterRoleBinding(conf ControllerConf) *rbacv1.ClusterRoleBinding { wantSubjects := []rbacv1.Subject{ { Kind: "ServiceAccount", Name: monov1alpha1.ControllerName, Namespace: conf.Namespace, }, } wantRoleRef := rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: monov1alpha1.ControllerName, } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{ APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding", }, ObjectMeta: metav1.ObjectMeta{ Name: monov1alpha1.ControllerName, Labels: conf.Labels, }, Subjects: wantSubjects, RoleRef: wantRoleRef, } } func buildControllerDeployment(conf ControllerConf) *appsv1.Deployment { replicas := int32(1) selectorLabels := map[string]string{ "app.kubernetes.io/name": monov1alpha1.ControllerName, "app.kubernetes.io/component": "controller", } podLabels := mergeStringMaps(conf.Labels, selectorLabels) runAsNonRoot := true allowPrivilegeEscalation := false userGroup := int64(65532) image, pullPolicy := controllerImage(conf) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ Name: monov1alpha1.ControllerName, Namespace: conf.Namespace, Labels: conf.Labels, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: podLabels, }, Spec: corev1.PodSpec{ ServiceAccountName: monov1alpha1.ControllerName, Containers: []corev1.Container{ { Name: "controller", Image: image, ImagePullPolicy: pullPolicy, Args: []string{ "controller", "--namespace", conf.Namespace, }, Env: []corev1.EnvVar{ { Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ APIVersion: "v1", FieldPath: "metadata.name", }, }, }, { Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ APIVersion: "v1", FieldPath: "metadata.namespace", }, }, }, { Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ APIVersion: "v1", FieldPath: "spec.nodeName", }, }, }, }, Ports: []corev1.ContainerPort{ { Name: "http", ContainerPort: 8080, Protocol: corev1.ProtocolTCP, }, { Name: "https", ContainerPort: 8443, Protocol: corev1.ProtocolTCP, }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.FromString("http"), }, }, InitialDelaySeconds: 5, PeriodSeconds: 60, TimeoutSeconds: 2, FailureThreshold: 3, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", Port: intstr.FromString("http"), }, }, InitialDelaySeconds: 2, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 3, }, SecurityContext: &corev1.SecurityContext{ RunAsNonRoot: &runAsNonRoot, RunAsUser: &userGroup, RunAsGroup: &userGroup, AllowPrivilegeEscalation: &allowPrivilegeEscalation, }, }, }, NodeSelector: controllerNodeSelector(conf), Affinity: controllerAffinity(conf), }, }, }, } } func controllerImage(conf ControllerConf) (string, corev1.PullPolicy) { if conf.Image != "" { return conf.Image, corev1.PullIfNotPresent } return fmt.Sprintf("localhost/monok8s/node-control:%s", buildinfo.Version), corev1.PullNever } func controllerNodeSelector(conf ControllerConf) map[string]string { if conf.Image != "" { return nil } // Local image exists on managed nodes only. return map[string]string{ monov1alpha1.NodeControlKey: "true", } } func controllerAffinity(conf ControllerConf) *corev1.Affinity { // Local image exists only on managed nodes, so in that mode we already use // NodeSelector and should not fight placement with anti-affinity. if conf.Image == "" { return nil } return &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ { Weight: 100, PodAffinityTerm: corev1.PodAffinityTerm{ TopologyKey: corev1.LabelHostname, LabelSelector: &metav1.LabelSelector{ MatchLabels: monov1alpha1.NodeAgentLabels(), }, }, }, }, }, } } func mergeStringMaps(maps ...map[string]string) map[string]string { var total int for _, m := range maps { total += len(m) } if total == 0 { return nil } out := make(map[string]string, total) for _, m := range maps { for k, v := range m { out[k] = v } } return out }