Drop admission logic. Use a plain controller instead
This commit is contained in:
@@ -1,157 +0,0 @@
|
||||
package admission
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
|
||||
"example.com/monok8s/pkg/controller/osupgrade"
|
||||
"example.com/monok8s/pkg/kube"
|
||||
|
||||
"github.com/emicklei/go-restful/v3"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apiserver/pkg/server/httplog"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
scheme = runtime.NewScheme()
|
||||
codecs = serializer.NewCodecFactory(scheme)
|
||||
deserializer = codecs.UniversalDeserializer()
|
||||
)
|
||||
|
||||
var statusesNoTracePred = httplog.StatusIsNot(
|
||||
http.StatusOK,
|
||||
http.StatusFound,
|
||||
http.StatusMovedPermanently,
|
||||
http.StatusTemporaryRedirect,
|
||||
http.StatusBadRequest,
|
||||
http.StatusNotFound,
|
||||
http.StatusSwitchingProtocols,
|
||||
)
|
||||
|
||||
func init() {
|
||||
_ = admissionv1.AddToScheme(scheme)
|
||||
_ = monov1alpha1.AddToScheme(scheme)
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
restfulCont *restful.Container
|
||||
|
||||
ctx context.Context
|
||||
clients *kube.Clients
|
||||
namespace string
|
||||
nodeName string
|
||||
}
|
||||
|
||||
func NewServer(ctx context.Context, clients *kube.Clients, namespace, nodeName string) *Server {
|
||||
s := &Server{
|
||||
ctx: ctx,
|
||||
clients: clients,
|
||||
namespace: namespace,
|
||||
nodeName: nodeName,
|
||||
}
|
||||
s.Initialize()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
if s == nil {
|
||||
http.Error(w, "admission server is nil", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if s.restfulCont == nil {
|
||||
http.Error(w, "admission server not initialized", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
handler := httplog.WithLogging(s.restfulCont, statusesNoTracePred)
|
||||
handler.ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
func (s *Server) Initialize() {
|
||||
s.restfulCont = restful.NewContainer()
|
||||
|
||||
ws := new(restful.WebService)
|
||||
|
||||
ws.Path("/admission").
|
||||
Consumes(restful.MIME_JSON).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.POST("").To(s.triggerAdmission).
|
||||
Reads(admissionv1.AdmissionReview{}).
|
||||
Writes(admissionv1.AdmissionReview{}))
|
||||
|
||||
s.restfulCont.Add(ws)
|
||||
}
|
||||
|
||||
func (s *Server) triggerAdmission(request *restful.Request, response *restful.Response) {
|
||||
body, err := io.ReadAll(request.Request.Body)
|
||||
if err != nil {
|
||||
_ = response.WriteError(http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
var reviewReq admissionv1.AdmissionReview
|
||||
if _, _, err := deserializer.Decode(body, nil, &reviewReq); err != nil {
|
||||
_ = response.WriteError(http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if reviewReq.Request == nil {
|
||||
_ = response.WriteErrorString(http.StatusBadRequest, "missing admission request")
|
||||
return
|
||||
}
|
||||
|
||||
resp := admissionv1.AdmissionReview{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "AdmissionReview",
|
||||
APIVersion: "admission.k8s.io/v1",
|
||||
},
|
||||
Response: &admissionv1.AdmissionResponse{
|
||||
UID: reviewReq.Request.UID,
|
||||
Allowed: true,
|
||||
Result: &metav1.Status{Message: "OK"},
|
||||
},
|
||||
}
|
||||
|
||||
var osu monov1alpha1.OSUpgrade
|
||||
if _, _, err := deserializer.Decode(reviewReq.Request.Object.Raw, nil, &osu); err != nil {
|
||||
klog.V(1).InfoS("Skipping non-OSUpgrade resource",
|
||||
"uid", reviewReq.Request.UID,
|
||||
"kind", reviewReq.Request.Kind.Kind,
|
||||
"operation", reviewReq.Request.Operation,
|
||||
"err", err,
|
||||
)
|
||||
_ = response.WriteEntity(resp)
|
||||
return
|
||||
}
|
||||
|
||||
klog.InfoS("Received OSUpgrade admission",
|
||||
"uid", reviewReq.Request.UID,
|
||||
"operation", reviewReq.Request.Operation,
|
||||
"name", osu.Name,
|
||||
"namespace", osu.Namespace,
|
||||
"node", s.nodeName,
|
||||
)
|
||||
|
||||
// Resolve every node name
|
||||
if err := osupgrade.EnsureOSUpgradeProgressForNode(
|
||||
s.ctx,
|
||||
s.clients,
|
||||
s.namespace,
|
||||
s.nodeName,
|
||||
&osu,
|
||||
); err != nil {
|
||||
klog.ErrorS(err, "ensure OSUpgradeProgress for node failed",
|
||||
"osupgrade", osu.Name,
|
||||
"node", s.nodeName,
|
||||
)
|
||||
}
|
||||
|
||||
_ = response.WriteEntity(resp)
|
||||
}
|
||||
346
clitools/pkg/controller/osupgrade/watch.go
Normal file
346
clitools/pkg/controller/osupgrade/watch.go
Normal file
@@ -0,0 +1,346 @@
|
||||
package osupgrade
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
|
||||
"example.com/monok8s/pkg/kube"
|
||||
)
|
||||
|
||||
func Watch(ctx context.Context, clients *kube.Clients, namespace string) error {
|
||||
var resourceVersion string
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
err := watchOnce(ctx, clients, namespace, &resourceVersion)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Expired RV is normal enough; clear it and relist.
|
||||
if apierrors.IsResourceExpired(err) {
|
||||
klog.InfoS("OSUpgrade watch resourceVersion expired; resetting",
|
||||
"namespace", namespace,
|
||||
"resourceVersion", resourceVersion,
|
||||
)
|
||||
resourceVersion = ""
|
||||
} else {
|
||||
klog.ErrorS(err, "OSUpgrade watch failed; retrying",
|
||||
"namespace", namespace,
|
||||
"resourceVersion", resourceVersion,
|
||||
)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func watchOnce(
|
||||
ctx context.Context,
|
||||
clients *kube.Clients,
|
||||
namespace string,
|
||||
resourceVersion *string,
|
||||
) error {
|
||||
// Cold start: list existing objects once, handle them, then watch from list RV.
|
||||
if *resourceVersion == "" {
|
||||
list, err := clients.MonoKS.
|
||||
Monok8sV1alpha1().
|
||||
OSUpgrades(namespace).
|
||||
List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list OSUpgrades: %w", err)
|
||||
}
|
||||
|
||||
for i := range list.Items {
|
||||
osu := &list.Items[i]
|
||||
handled, err := handleOSUpgrade(ctx, clients, namespace, osu)
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "reconcile existing OSUpgrade failed",
|
||||
"name", osu.Name,
|
||||
"resourceVersion", osu.ResourceVersion,
|
||||
)
|
||||
continue
|
||||
}
|
||||
if !handled {
|
||||
klog.V(2).InfoS("skipping existing OSUpgrade",
|
||||
"name", osu.Name,
|
||||
"phase", osu.StatusPhase(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
*resourceVersion = list.ResourceVersion
|
||||
klog.InfoS("initial OSUpgrade sync complete",
|
||||
"namespace", namespace,
|
||||
"resourceVersion", *resourceVersion,
|
||||
"count", len(list.Items),
|
||||
)
|
||||
}
|
||||
|
||||
w, err := clients.MonoKS.
|
||||
Monok8sV1alpha1().
|
||||
OSUpgrades(namespace).
|
||||
Watch(ctx, metav1.ListOptions{
|
||||
ResourceVersion: *resourceVersion,
|
||||
AllowWatchBookmarks: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("watch OSUpgrades: %w", err)
|
||||
}
|
||||
defer w.Stop()
|
||||
|
||||
klog.InfoS("watching OSUpgrades",
|
||||
"namespace", namespace,
|
||||
"resourceVersion", *resourceVersion,
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
|
||||
case evt, ok := <-w.ResultChan():
|
||||
if !ok {
|
||||
return fmt.Errorf("watch channel closed")
|
||||
}
|
||||
|
||||
switch evt.Type {
|
||||
case watch.Bookmark:
|
||||
if rv := extractResourceVersion(evt.Object); rv != "" {
|
||||
*resourceVersion = rv
|
||||
}
|
||||
continue
|
||||
|
||||
case watch.Error:
|
||||
// Let outer loop retry / relist.
|
||||
return fmt.Errorf("watch returned error event")
|
||||
|
||||
case watch.Deleted:
|
||||
// Top-level delete does not require action here.
|
||||
continue
|
||||
|
||||
case watch.Added, watch.Modified:
|
||||
// handled below
|
||||
|
||||
default:
|
||||
klog.V(1).InfoS("skipping unexpected watch event type",
|
||||
"eventType", evt.Type,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
osu, ok := evt.Object.(*monov1alpha1.OSUpgrade)
|
||||
if !ok {
|
||||
klog.V(1).InfoS("skipping unexpected watch object type",
|
||||
"type", fmt.Sprintf("%T", evt.Object),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if osu.ResourceVersion != "" {
|
||||
*resourceVersion = osu.ResourceVersion
|
||||
}
|
||||
|
||||
handled, err := handleOSUpgrade(ctx, clients, namespace, osu)
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "reconcile OSUpgrade failed",
|
||||
"name", osu.Name,
|
||||
"eventType", evt.Type,
|
||||
"resourceVersion", osu.ResourceVersion,
|
||||
)
|
||||
continue
|
||||
}
|
||||
if !handled {
|
||||
klog.V(2).InfoS("skipping OSUpgrade",
|
||||
"name", osu.Name,
|
||||
"eventType", evt.Type,
|
||||
"phase", osu.StatusPhase(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleOSUpgrade(
|
||||
ctx context.Context,
|
||||
clients *kube.Clients,
|
||||
namespace string,
|
||||
osu *monov1alpha1.OSUpgrade,
|
||||
) (bool, error) {
|
||||
if !shouldHandle(osu) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if osu.Status == nil || osu.Status.ObservedGeneration != osu.Generation {
|
||||
return true, reconcileSpec(ctx, clients, namespace, osu)
|
||||
}
|
||||
|
||||
if osu.Status.Phase == monov1alpha1.OSUpgradePhaseAccepted {
|
||||
return true, reconcileFanout(ctx, clients, namespace, osu)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func reconcileSpec(
|
||||
ctx context.Context,
|
||||
clients *kube.Clients,
|
||||
namespace string,
|
||||
osu *monov1alpha1.OSUpgrade,
|
||||
) error {
|
||||
osu = osu.DeepCopy()
|
||||
|
||||
osu.Status = &monov1alpha1.OSUpgradeStatus{
|
||||
Phase: monov1alpha1.OSUpgradePhaseAccepted,
|
||||
ResolvedVersion: osu.Spec.DesiredVersion,
|
||||
ObservedGeneration: osu.Generation,
|
||||
}
|
||||
|
||||
_, err := clients.MonoKS.
|
||||
Monok8sV1alpha1().
|
||||
OSUpgrades(namespace).
|
||||
UpdateStatus(ctx, osu, metav1.UpdateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func reconcileFanout(
|
||||
ctx context.Context,
|
||||
clients *kube.Clients,
|
||||
namespace string,
|
||||
osu *monov1alpha1.OSUpgrade,
|
||||
) error {
|
||||
nodeNames, err := listTargetNodeNames(ctx, clients, osu)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list target nodes for %s: %w", osu.Name, err)
|
||||
}
|
||||
|
||||
if len(nodeNames) == 0 {
|
||||
klog.InfoS("no targets", "osupgrade", osu.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
klog.InfoS("ensuring OSUpgradeProgress for target nodes",
|
||||
"osupgrade", osu.Name,
|
||||
"targets", len(nodeNames),
|
||||
)
|
||||
|
||||
for _, nodeName := range nodeNames {
|
||||
if err := EnsureOSUpgradeProgressForNode(
|
||||
ctx,
|
||||
clients,
|
||||
namespace,
|
||||
nodeName,
|
||||
osu,
|
||||
); err != nil {
|
||||
klog.ErrorS(err, "ensure OSUpgradeProgress for node failed",
|
||||
"osupgrade", osu.Name,
|
||||
"node", nodeName,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func listTargetNodeNames(
|
||||
ctx context.Context,
|
||||
clients *kube.Clients,
|
||||
osu *monov1alpha1.OSUpgrade,
|
||||
) ([]string, error) {
|
||||
selector := labels.SelectorFromSet(labels.Set{
|
||||
monov1alpha1.ControlAgentKey: "true",
|
||||
})
|
||||
|
||||
if osu.Spec.NodeSelector != nil {
|
||||
sel, err := metav1.LabelSelectorAsSelector(osu.Spec.NodeSelector)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid nodeSelector: %w", err)
|
||||
}
|
||||
|
||||
reqs, selectable := sel.Requirements()
|
||||
if !selectable {
|
||||
return nil, fmt.Errorf("nodeSelector is not selectable")
|
||||
}
|
||||
|
||||
selector = selector.Add(reqs...)
|
||||
}
|
||||
|
||||
list, err := clients.Kubernetes.CoreV1().
|
||||
Nodes().
|
||||
List(ctx, metav1.ListOptions{
|
||||
LabelSelector: selector.String(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list nodes: %w", err)
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(list.Items))
|
||||
for i := range list.Items {
|
||||
node := &list.Items[i]
|
||||
if shouldUseNode(node) {
|
||||
out = append(out, node.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func shouldUseNode(node *corev1.Node) bool {
|
||||
// Keep this conservative for now. Tighten if you want only Ready nodes.
|
||||
return node != nil && node.Name != ""
|
||||
}
|
||||
|
||||
func shouldHandle(osu *monov1alpha1.OSUpgrade) bool {
|
||||
if osu == nil || osu.DeletionTimestamp != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if osu.Spec.DesiredVersion == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// NEW: initial processing stage
|
||||
if osu.Status == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// Reconcile if spec changed
|
||||
if osu.Status.ObservedGeneration != osu.Generation {
|
||||
return true
|
||||
}
|
||||
|
||||
// Fanout stage
|
||||
return osu.Status.Phase == monov1alpha1.OSUpgradePhaseAccepted
|
||||
}
|
||||
|
||||
func extractResourceVersion(obj interface{}) string {
|
||||
type hasRV interface {
|
||||
GetResourceVersion() string
|
||||
}
|
||||
|
||||
if o, ok := obj.(hasRV); ok {
|
||||
return o.GetResourceVersion()
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
92
clitools/pkg/controller/server.go
Normal file
92
clitools/pkg/controller/server.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"example.com/monok8s/pkg/kube"
|
||||
|
||||
"github.com/emicklei/go-restful/v3"
|
||||
"k8s.io/apiserver/pkg/server/httplog"
|
||||
)
|
||||
|
||||
var statusesNoTracePred = httplog.StatusIsNot(
|
||||
http.StatusOK,
|
||||
http.StatusFound,
|
||||
http.StatusMovedPermanently,
|
||||
http.StatusTemporaryRedirect,
|
||||
http.StatusBadRequest,
|
||||
http.StatusNotFound,
|
||||
http.StatusSwitchingProtocols,
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
restfulCont *restful.Container
|
||||
|
||||
ctx context.Context
|
||||
clients *kube.Clients
|
||||
namespace string
|
||||
nodeName string
|
||||
startedAt time.Time
|
||||
}
|
||||
|
||||
type StatusResponse struct {
|
||||
OK bool `json:"ok"`
|
||||
Service string `json:"service"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"`
|
||||
UptimeSec int64 `json:"uptimeSec"`
|
||||
}
|
||||
|
||||
func NewServer(ctx context.Context, clients *kube.Clients, namespace, nodeName string) *Server {
|
||||
s := &Server{
|
||||
ctx: ctx,
|
||||
clients: clients,
|
||||
namespace: namespace,
|
||||
nodeName: nodeName,
|
||||
startedAt: time.Now(),
|
||||
}
|
||||
s.Initialize()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
if s == nil {
|
||||
http.Error(w, "server is nil", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if s.restfulCont == nil {
|
||||
http.Error(w, "server not initialized", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
handler := httplog.WithLogging(s.restfulCont, statusesNoTracePred)
|
||||
handler.ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
func (s *Server) Initialize() {
|
||||
s.restfulCont = restful.NewContainer()
|
||||
|
||||
ws := new(restful.WebService)
|
||||
ws.Path("/")
|
||||
ws.Consumes(restful.MIME_JSON)
|
||||
ws.Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET("/status").To(s.queryStatus).
|
||||
Doc("Return basic controller status"))
|
||||
|
||||
s.restfulCont.Add(ws)
|
||||
}
|
||||
|
||||
func (s *Server) queryStatus(request *restful.Request, response *restful.Response) {
|
||||
resp := StatusResponse{
|
||||
OK: true,
|
||||
Service: "monok8s-controller",
|
||||
Namespace: s.namespace,
|
||||
NodeName: s.nodeName,
|
||||
UptimeSec: int64(time.Since(s.startedAt).Seconds()),
|
||||
}
|
||||
|
||||
_ = response.WriteHeaderAndEntity(http.StatusOK, resp)
|
||||
}
|
||||
Reference in New Issue
Block a user