diff --git a/build.env b/build.env index 585b280..970850f 100644 --- a/build.env +++ b/build.env @@ -43,3 +43,7 @@ BUILD_TAG=MONOK8S # Optional apt cache # example: apt-cacher-ng.eco-system.svc.cluster.local:3142 APT_PROXY= + +# remote image repository prefix to push to +# e.g. ghcr.io/monok8s +IMAGE_REPOSITORY= diff --git a/clitools/docker/ctl-agent.Dockerfile b/clitools/docker/ctl-agent.Dockerfile index 79766a0..a11e39e 100644 --- a/clitools/docker/ctl-agent.Dockerfile +++ b/clitools/docker/ctl-agent.Dockerfile @@ -1,16 +1,41 @@ +ARG BASE_IMAGE=localhost/monok8s/ctl-build-base:dev + +FROM --platform=$BUILDPLATFORM ${BASE_IMAGE} AS build + +ARG VERSION=dev +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src + +COPY . . + +RUN test -f pkg/buildinfo/buildinfo_gen.go + +RUN mkdir -p /out && \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + go build -trimpath -ldflags="-s -w" \ + -o /out/ctl ./cmd/ctl + + FROM alpine:latest AS cacerts + + FROM scratch ARG VERSION +ARG TARGETOS +ARG TARGETARCH ENV VERSION=${VERSION} WORKDIR / -COPY bin/ctl-linux-aarch64-${VERSION} ./ctl -COPY out/fw_printenv ./ -COPY out/fw_setenv ./ +COPY --from=build /out/ctl /ctl COPY --from=cacerts /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY out/uboot-tools/${TARGETOS}_${TARGETARCH}/fw_printenv /fw_printenv +COPY out/uboot-tools/${TARGETOS}_${TARGETARCH}/fw_setenv /fw_setenv + ENV PATH=/ ENTRYPOINT ["/ctl"] diff --git a/clitools/docker/ctl-builder.Dockerfile b/clitools/docker/ctl-builder.Dockerfile deleted file mode 100644 index 3e171c6..0000000 --- a/clitools/docker/ctl-builder.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG BASE_IMAGE=localhost/monok8s/ctl-build-base:dev -FROM ${BASE_IMAGE} AS build - -ARG VERSION=dev -ARG TARGETOS=linux -ARG TARGETARCH=arm64 - -WORKDIR /src - -COPY . . - -RUN test -f pkg/buildinfo/buildinfo_gen.go - -RUN mkdir -p /out && \ - GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ - go build -trimpath -ldflags="-s -w" \ - -o /out/ctl-linux-aarch64-${VERSION} ./cmd/ctl - -FROM scratch -COPY --from=build /out/ / diff --git a/clitools/makefile b/clitools/makefile index 69f7950..58580c9 100644 --- a/clitools/makefile +++ b/clitools/makefile @@ -1,4 +1,6 @@ include ../build.env +-include ../build.env.work +export BUILD_PLATFORM ?= linux/amd64 @@ -11,21 +13,24 @@ KUBE_VERSION ?= v1.33.3 GIT_REV := $(shell git rev-parse HEAD) -PACKAGES_DIR := packages -BIN_DIR := bin -OUT_DIR := out +PACKAGES_DIR := packages +BIN_DIR := bin +OUT_DIR := out +UBOOT_TOOLS_OUT := $(OUT_DIR)/uboot-tools UBOOT_TAR := $(PACKAGES_DIR)/uboot-$(UBOOT_VERSION).tar.gz BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go -CRD_PATHS := ./pkg/apis/... - +ASSETS_PATH := ./pkg/assets BUILDX_BUILDER := container-builder LOCAL_REGISTRY := registry LOCAL_REGISTRY_PORT := 5000 -CTL_BUILD_BASE_IMAGE := localhost:5000/monok8s/ctl-build-base:$(VERSION) -CTL_BINARY := ctl-linux-aarch64-$(VERSION) +CTL_BUILD_BASE_REPO := localhost:5000/monok8s/ctl-build-base +CTL_IMAGE_REPO := localhost:5000/monok8s/node-control + +CTL_BUILD_BASE_IMAGE := $(CTL_BUILD_BASE_REPO):$(VERSION) +CTL_IMAGE := $(CTL_IMAGE_REPO):$(VERSION) DOWNLOAD_PACKAGES_STAMP := $(PACKAGES_DIR)/.download-packages.stamp @@ -88,11 +93,14 @@ $(DOWNLOAD_PACKAGES_STAMP): docker/download-packages.Dockerfile makefile | $(PAC @touch $@ uboot-tools: $(DOWNLOAD_PACKAGES_STAMP) - docker buildx build --platform linux/arm64 \ + rm -rf "$(UBOOT_TOOLS_OUT)" + mkdir -p "$(UBOOT_TOOLS_OUT)" + docker buildx build \ + --platform linux/amd64,linux/arm64 \ -f docker/uboot-tools.Dockerfile \ --build-arg UBOOT_VERSION=$(UBOOT_VERSION) \ --build-arg UBOOT_TAR=$(UBOOT_TAR) \ - --output type=local,dest=./$(OUT_DIR) . + --output type=local,dest=./$(UBOOT_TOOLS_OUT),platform-split=true . ctl-build-base: ensure-buildx ensure-registry docker buildx build \ @@ -101,16 +109,6 @@ ctl-build-base: ensure-buildx ensure-registry -t $(CTL_BUILD_BASE_IMAGE) \ --output type=image,push=true,registry.insecure=true . -build-bin: .buildinfo ctl-build-base | $(BIN_DIR) - docker buildx build \ - --platform $(BUILD_PLATFORM) \ - -f docker/ctl-builder.Dockerfile \ - --build-arg BASE_IMAGE=$(CTL_BUILD_BASE_IMAGE) \ - --build-arg VERSION=$(VERSION) \ - --build-arg TARGETOS=linux \ - --build-arg TARGETARCH=arm64 \ - --output type=local,dest=./$(BIN_DIR) . - build-crds: ctl-build-base | $(OUT_DIR) mkdir -p "$(OUT_DIR)/crds" docker buildx build \ @@ -118,16 +116,20 @@ build-crds: ctl-build-base | $(OUT_DIR) -f docker/crdgen.Dockerfile \ --build-arg BASE_IMAGE=$(CTL_BUILD_BASE_IMAGE) \ --output type=local,dest=./$(OUT_DIR)/crds . + rm -rf "$(ASSETS_PATH)/crds" + mkdir -p "$(ASSETS_PATH)/crds" + cp -R "$(OUT_DIR)/crds/." "$(ASSETS_PATH)/crds/" -build-agent: build uboot-tools +build-agent: .buildinfo build-crds uboot-tools docker buildx build \ - --platform linux/arm64 \ + --platform linux/amd64,linux/arm64 \ -f docker/ctl-agent.Dockerfile \ + --build-arg BASE_IMAGE=$(CTL_BUILD_BASE_IMAGE) \ --build-arg VERSION=$(VERSION) \ - --load \ - -t localhost/monok8s/node-control:$(VERSION) . + -t $(CTL_IMAGE) \ + --output type=image,push=true,registry.insecure=true . -build-local: .buildinfo | $(BIN_DIR) +build-local: build-crds .buildinfo | $(BIN_DIR) docker buildx build \ -f docker/ctl-builder-local.Dockerfile \ --build-arg VERSION=$(VERSION) \ @@ -135,14 +137,22 @@ build-local: .buildinfo | $(BIN_DIR) --build-arg GIT_REV=$(GIT_REV) \ --output type=local,dest=./$(BIN_DIR) . +push-agent: .buildinfo build-crds uboot-tools + test -n "$(IMAGE_REPOSITORY)" + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -f docker/ctl-agent.Dockerfile \ + --build-arg BASE_IMAGE=$(CTL_BUILD_BASE_IMAGE) \ + --build-arg VERSION=$(VERSION) \ + -t $(IMAGE_REPOSITORY)/node-control:$(VERSION) \ + --push . + run-agent: docker run --rm \ -v "$$(pwd)/out:/work/out" \ - localhost/monok8s/node-control:$(VERSION) \ + $(CTL_IMAGE) \ agent --env-file /work/out/cluster.env -build: build-bin build-crds - clean: -docker image rm localhost/monok8s/node-control:$(VERSION) >/dev/null 2>&1 || true rm -rf \ @@ -158,7 +168,6 @@ dockerclean: - docker rmi \ localhost/monok8s/ctl-build-base:$(VERSION) \ localhost/monok8s/node-control:$(VERSION) \ - localhost/monok8s/ctl-builder:$(VERSION) \ localhost/monok8s/crdgen:$(VERSION) \ 2>/dev/null || true @@ -169,10 +178,10 @@ dockerclean: pkgclean: rm -rf $(PACKAGES_DIR) -all: build build-agent build-local +all: build-agent build-local .PHONY: \ all clean dockerclean \ .buildinfo ensure-buildx ensure-registry \ - build build-bin build-crds build-local build-agent \ - uboot-tools run-agent + build-crds build-local build-agent build-agent-local push-agent \ + uboot-tools run-agent run-agent-local diff --git a/clitools/pkg/assets/crds/monok8s.io_monoksconfigs.yaml b/clitools/pkg/assets/crds/monok8s.io_monoksconfigs.yaml new file mode 100644 index 0000000..cfdf572 --- /dev/null +++ b/clitools/pkg/assets/crds/monok8s.io_monoksconfigs.yaml @@ -0,0 +1,178 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: monoksconfigs.monok8s.io +spec: + group: monok8s.io + names: + kind: MonoKSConfig + listKind: MonoKSConfigList + plural: monoksconfigs + singular: monoksconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + allowSchedulingOnControlPlane: + type: boolean + apiServerAdvertiseAddress: + type: string + apiServerEndpoint: + type: string + bootstrapToken: + type: string + clusterDomain: + type: string + clusterName: + type: string + clusterRole: + type: string + cniPlugin: + type: string + containerRuntimeEndpoint: + type: string + controlPlaneCertKey: + type: string + discoveryTokenCACertHash: + type: string + enableNodeControl: + type: boolean + initControlPlane: + type: boolean + kubeProxyNodePortAddresses: + items: + type: string + type: array + kubernetesVersion: + type: string + network: + properties: + dnsNameservers: + items: + type: string + type: array + dnsSearchDomains: + items: + type: string + type: array + hostname: + type: string + managementCIDR: + type: string + managementGateway: + type: string + managementIface: + type: string + type: object + nodeLabels: + additionalProperties: + type: string + type: object + nodeName: + type: string + podSubnet: + type: string + serviceSubnet: + type: string + skipImageCheck: + type: boolean + subjectAltNames: + items: + type: string + type: array + type: object + status: + properties: + appliedSteps: + items: + type: string + type: array + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + format: int64 + type: integer + phase: + type: string + type: object + type: object + served: true + storage: true diff --git a/clitools/pkg/assets/crds/monok8s.io_osupgradeprogresses.yaml b/clitools/pkg/assets/crds/monok8s.io_osupgradeprogresses.yaml new file mode 100644 index 0000000..a216245 --- /dev/null +++ b/clitools/pkg/assets/crds/monok8s.io_osupgradeprogresses.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: osupgradeprogresses.monok8s.io +spec: + group: monok8s.io + names: + kind: OSUpgradeProgress + listKind: OSUpgradeProgressList + plural: osupgradeprogresses + shortNames: + - osup + singular: osupgradeprogress + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.nodeName + name: Node + type: string + - jsonPath: .spec.sourceRef.name + name: Source + type: string + - jsonPath: .status.currentVersion + name: Current + type: string + - jsonPath: .status.targetVersion + name: Target + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the OSUpgradeProgress. + properties: + nodeName: + type: string + retryNonce: + description: |- + RetryNonce triggers a retry when its value changes. + Users can update this field (for example, set it to the current time) + to request a retry of a failed OS upgrade. + type: string + sourceRef: + properties: + name: + type: string + namespace: + type: string + type: object + type: object + status: + description: Most recently observed status of the OSUpgradeProgress. + properties: + completedAt: + format: date-time + type: string + currentFrom: + type: string + currentStep: + format: int32 + type: integer + currentTo: + type: string + currentVersion: + type: string + failureReason: + type: string + inactivePartition: + type: string + lastUpdatedAt: + format: date-time + type: string + message: + type: string + observedRetryNonce: + description: |- + ObservedRetryNonce records the last retryNonce value the agent accepted. + When spec.retryNonce is changed by the user and differs from this value, + the agent may retry a failed upgrade. + type: string + phase: + type: string + plannedPath: + items: + type: string + type: array + retryCount: + format: int32 + type: integer + startedAt: + format: date-time + type: string + targetVersion: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/clitools/pkg/assets/crds/monok8s.io_osupgrades.yaml b/clitools/pkg/assets/crds/monok8s.io_osupgrades.yaml new file mode 100644 index 0000000..2c0ebb7 --- /dev/null +++ b/clitools/pkg/assets/crds/monok8s.io_osupgrades.yaml @@ -0,0 +1,202 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: osupgrades.monok8s.io +spec: + group: monok8s.io + names: + kind: OSUpgrade + listKind: OSUpgradeList + plural: osupgrades + shortNames: + - osu + singular: osupgrade + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.desiredVersion + name: Desired + type: string + - jsonPath: .status.resolvedVersion + name: Resolved + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the OSUpgrade. + properties: + catalog: + properties: + configMapRef: + type: string + inline: + type: string + url: + type: string + type: object + desiredVersion: + minLength: 1 + type: string + flashProfile: + default: balanced + description: |- + Profiles (TODO) + safe - api-server can be responsive most of the time + balanced - api-server can sometimes be unresponsive + fast - disable throttling. Good for worker node. + enum: + - fast + - balanced + - safe + type: string + nodeSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + status: + description: Most recently observed status of the OSUpgrade. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + reason: + type: string + resolvedVersion: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/clitools/pkg/assets/embed.go b/clitools/pkg/assets/embed.go new file mode 100644 index 0000000..c47ab7f --- /dev/null +++ b/clitools/pkg/assets/embed.go @@ -0,0 +1,6 @@ +package assets + +import "embed" + +//go:embed crds/*.yaml +var CRDs embed.FS diff --git a/clitools/pkg/assets/render.go b/clitools/pkg/assets/render.go new file mode 100644 index 0000000..7c6b967 --- /dev/null +++ b/clitools/pkg/assets/render.go @@ -0,0 +1,49 @@ +package assets + +import ( + "fmt" + "io" + "path/filepath" + "sort" +) + +func PrintCRDs(out io.Writer) error { + entries, err := CRDs.ReadDir("crds") + if err != nil { + return err + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if filepath.Ext(entry.Name()) != ".yaml" { + continue + } + names = append(names, entry.Name()) + } + + sort.Strings(names) + + for _, name := range names { + b, err := CRDs.ReadFile("crds/" + name) + if err != nil { + return err + } + + if _, err := fmt.Fprintln(out, "---"); err != nil { + return err + } + if _, err := out.Write(b); err != nil { + return err + } + if len(b) == 0 || b[len(b)-1] != '\n' { + if _, err := fmt.Fprintln(out); err != nil { + return err + } + } + } + + return nil +} diff --git a/clitools/pkg/bootstrap/runner.go b/clitools/pkg/bootstrap/runner.go index 18797d3..48167b2 100644 --- a/clitools/pkg/bootstrap/runner.go +++ b/clitools/pkg/bootstrap/runner.go @@ -112,6 +112,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner { Name: "Wait for existing cluster", Desc: "Block until control plane is reachable when joining or reconciling an existing cluster", }, + { + RegKey: "CheckForVersionSkew", + Name: "Check for version skew", + Desc: "Validate wether version satisfy the requirements againts current cluster if any", + }, { RegKey: "ReconcileControlPlane", Name: "Reconcile control plane", @@ -122,11 +127,6 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner { Name: "Reconcile worker node", Desc: "Reconcile the worker node", }, - { - RegKey: "CheckForVersionSkew", - Name: "Check for version skew", - Desc: "Validate wether version satisfy the requirements againts current cluster if any", - }, { RegKey: "RunKubeadmUpgradeApply", Name: "Run kubeadm upgrade apply", diff --git a/clitools/pkg/cmd/create/create.go b/clitools/pkg/cmd/create/create.go index b450c45..e3b5fd8 100644 --- a/clitools/pkg/cmd/create/create.go +++ b/clitools/pkg/cmd/create/create.go @@ -6,7 +6,9 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" "os" + "strings" + assets "example.com/monok8s/pkg/assets" render "example.com/monok8s/pkg/render" ) @@ -42,13 +44,20 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command { return err }, }, + &cobra.Command{ + Use: "crds", + Short: "Print the bundled CRDs", + RunE: func(cmd *cobra.Command, _ []string) error { + return assets.PrintCRDs(cmd.OutOrStdout()) + }, + }, ) var authorizedKeysPath string sshdcmd := cobra.Command{ Use: "sshd", - Short: "Print sshd deployment template", + Short: "Print sshd deployments template", RunE: func(cmd *cobra.Command, _ []string) error { ns, _, err := flags.ToRawKubeConfigLoader().Namespace() if err != nil { @@ -77,8 +86,12 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command { cconf := render.ControllerConf{} controllercmd := cobra.Command{ Use: "controller", - Short: "Print controller deployment template", + Short: "Print controller deployments template", RunE: func(cmd *cobra.Command, _ []string) error { + if len(cconf.ImagePullSecrets) > 0 && strings.TrimSpace(cconf.Image) == "" { + return fmt.Errorf("--image-pull-secret requires --image") + } + ns, _, err := flags.ToRawKubeConfigLoader().Namespace() if err != nil { return err @@ -102,9 +115,56 @@ func NewCmdCreate(flags *genericclioptions.ConfigFlags) *cobra.Command { "", "Controller image, including optional registry and tag", ) + controllercmd.Flags().StringSliceVar( + &cconf.ImagePullSecrets, + "image-pull-secret", + nil, + "Image pull secret name for the agent image; may be specified multiple times or as a comma-separated list", + ) cmd.AddCommand(&controllercmd) + aconf := render.AgentConf{} + agentcmd := cobra.Command{ + Use: "agent", + Short: "Print agent daemonsets template", + RunE: func(cmd *cobra.Command, _ []string) error { + if len(aconf.ImagePullSecrets) > 0 && strings.TrimSpace(aconf.Image) == "" { + return fmt.Errorf("--image-pull-secret requires --image") + } + + ns, _, err := flags.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + aconf.Namespace = ns + + out, err := render.RenderAgentDaemonSets(aconf) + if err != nil { + return err + } + + _, err = fmt.Fprint(cmd.OutOrStdout(), out) + return err + }, + } + + agentcmd.Flags().StringVar( + &aconf.Image, + "image", + "", + "Agent image, including optional registry and tag", + ) + agentcmd.Flags().StringSliceVar( + &aconf.ImagePullSecrets, + "image-pull-secret", + nil, + "Image pull secret name for the agent image; may be specified multiple times or as a comma-separated list", + ) + + cmd.AddCommand(&agentcmd) + return cmd } diff --git a/clitools/pkg/crds/crds.go b/clitools/pkg/crds/crds.go deleted file mode 100644 index 301e981..0000000 --- a/clitools/pkg/crds/crds.go +++ /dev/null @@ -1,76 +0,0 @@ -package crds - -import ( - monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Definitions() []*apiextensionsv1.CustomResourceDefinition { - return []*apiextensionsv1.CustomResourceDefinition{ - monoKSConfigCRD(), - osUpgradeCRD(), - } -} - -func monoKSConfigCRD() *apiextensionsv1.CustomResourceDefinition { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.MonoKSConfigCRD, - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: monov1alpha1.Group, - Scope: apiextensionsv1.NamespaceScoped, - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "monoksconfigs", - Singular: "monoksconfig", - Kind: "MonoKSConfig", - ShortNames: []string{"mkscfg"}, - }, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ - Name: "v1alpha1", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "spec": {Type: "object", XPreserveUnknownFields: boolPtr(true)}, - "status": {Type: "object", XPreserveUnknownFields: boolPtr(true)}, - }, - }}, - }}, - }, - } -} - -func osUpgradeCRD() *apiextensionsv1.CustomResourceDefinition { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.OSUpgradeCRD, - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: monov1alpha1.Group, - Scope: apiextensionsv1.NamespaceScoped, - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "osupgrades", - Singular: "osupgrade", - Kind: "OSUpgrade", - ShortNames: []string{"osup"}, - }, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ - Name: "v1alpha1", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "spec": {Type: "object", XPreserveUnknownFields: boolPtr(true)}, - "status": {Type: "object", XPreserveUnknownFields: boolPtr(true)}, - }, - }}, - }}, - }, - } -} - -func boolPtr(v bool) *bool { return &v } diff --git a/clitools/pkg/node/agent.go b/clitools/pkg/node/agent.go index 7f5acc4..1f84b62 100644 --- a/clitools/pkg/node/agent.go +++ b/clitools/pkg/node/agent.go @@ -3,37 +3,27 @@ package node import ( "context" "fmt" - "reflect" "strings" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" "example.com/monok8s/pkg/kube" + "example.com/monok8s/pkg/render" templates "example.com/monok8s/pkg/templates" ) -const ( - controlAgentImage = "localhost/monok8s/node-control:dev" - kubeconfig = "/etc/kubernetes/admin.conf" -) +const kubeconfig = "/etc/kubernetes/admin.conf" func ApplyNodeControlDaemonSetResources(ctx context.Context, n *NodeContext) error { - // Only the control-plane should bootstrap this DaemonSet definition. - // And only when the feature is enabled. if strings.TrimSpace(n.Config.Spec.ClusterRole) != "control-plane" || !n.Config.Spec.EnableNodeControl { - klog.InfoS("skipped for", "clusterRole", n.Config.Spec.ClusterRole, "enableNodeAgent", n.Config.Spec.EnableNodeControl) + klog.InfoS("skipped for", + "clusterRole", n.Config.Spec.ClusterRole, + "enableNodeAgent", n.Config.Spec.EnableNodeControl, + ) return nil } - err := ApplyCRDs(ctx, n) - if err != nil { + if err := ApplyCRDs(ctx, n); err != nil { return err } @@ -47,363 +37,13 @@ func ApplyNodeControlDaemonSetResources(ctx context.Context, n *NodeContext) err return fmt.Errorf("build kube clients from %s: %w", kubeconfig, err) } - labels := map[string]string{ - "app.kubernetes.io/name": monov1alpha1.NodeAgentName, - "app.kubernetes.io/component": "agent", - "app.kubernetes.io/part-of": "monok8s", - "app.kubernetes.io/managed-by": monov1alpha1.NodeControlName, + conf := render.AgentConf{ + Namespace: namespace, } - kubeClient := clients.Kubernetes - - if err := ensureNamespace(ctx, kubeClient, namespace, labels); err != nil { - return fmt.Errorf("ensure namespace %q: %w", namespace, err) - } - if err := applyNodeAgentServiceAccount(ctx, kubeClient, namespace, labels); err != nil { - return fmt.Errorf("apply serviceaccount: %w", err) - } - if err := applyNodeAgentClusterRole(ctx, kubeClient, labels); err != nil { - return fmt.Errorf("apply clusterrole: %w", err) - } - if err := applyNodeAgentClusterRoleBinding(ctx, kubeClient, namespace, labels); err != nil { - return fmt.Errorf("apply clusterrolebinding: %w", err) - } - if err := applyNodeAgentDaemonSet(ctx, kubeClient, namespace, labels); err != nil { - return fmt.Errorf("apply daemonset: %w", err) + if err := render.ApplyAgentDaemonSets(ctx, clients.Kubernetes, conf); err != nil { + return fmt.Errorf("apply node agent daemonset resources: %w", err) } return nil } - -func ensureNamespace( - ctx context.Context, - kubeClient kubernetes.Interface, - namespace string, - labels map[string]string, -) error { - _, err := kubeClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) - if err == nil { - return nil - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get namespace: %w", err) - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: copyStringMap(labels), - }, - } - - _, err = kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("create namespace: %w", err) - } - - return nil -} - -func copyStringMap(in map[string]string) map[string]string { - if len(in) == 0 { - return nil - } - out := make(map[string]string, len(in)) - for k, v := range in { - out[k] = v - } - return out -} - -func applyNodeAgentServiceAccount(ctx context.Context, kubeClient kubernetes.Interface, namespace string, labels map[string]string) error { - want := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.NodeAgentName, - Namespace: namespace, - Labels: labels, - }, - } - - existing, err := kubeClient.CoreV1().ServiceAccounts(namespace).Get(ctx, monov1alpha1.NodeAgentName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = kubeClient.CoreV1().ServiceAccounts(namespace).Create(ctx, want, metav1.CreateOptions{}) - return err - } - if err != nil { - return err - } - - changed := false - if !reflect.DeepEqual(existing.Labels, want.Labels) { - existing.Labels = want.Labels - changed = true - } - - if !changed { - return nil - } - - _, err = kubeClient.CoreV1().ServiceAccounts(namespace).Update(ctx, existing, metav1.UpdateOptions{}) - return err -} - -func applyNodeAgentClusterRole(ctx context.Context, kubeClient kubernetes.Interface, labels map[string]string) error { - wantRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{monov1alpha1.Group}, - Resources: []string{"osupgrades"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{monov1alpha1.Group}, - Resources: []string{"osupgradeprogresses"}, - Verbs: []string{"get", "list", "watch", "create", "patch", "update"}, - }, - { - APIGroups: []string{monov1alpha1.Group}, - Resources: []string{"osupgradeprogresses/status"}, - Verbs: []string{"get", "list", "watch", "create", "patch", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - want := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.NodeAgentName, - Labels: labels, - }, - Rules: wantRules, - } - - existing, err := kubeClient.RbacV1().ClusterRoles().Get(ctx, monov1alpha1.NodeAgentName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = kubeClient.RbacV1().ClusterRoles().Create(ctx, want, metav1.CreateOptions{}) - return err - } - if err != nil { - return err - } - - changed := false - if !reflect.DeepEqual(existing.Labels, want.Labels) { - existing.Labels = want.Labels - changed = true - } - if !reflect.DeepEqual(existing.Rules, want.Rules) { - existing.Rules = want.Rules - changed = true - } - - if !changed { - return nil - } - - _, err = kubeClient.RbacV1().ClusterRoles().Update(ctx, existing, metav1.UpdateOptions{}) - return err -} - -func applyNodeAgentClusterRoleBinding(ctx context.Context, kubeClient kubernetes.Interface, namespace string, labels map[string]string) error { - wantRoleRef := rbacv1.RoleRef{ - APIGroup: rbacv1.GroupName, - Kind: "ClusterRole", - Name: monov1alpha1.NodeAgentName, - } - wantSubjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: monov1alpha1.NodeAgentName, - Namespace: namespace, - }, - } - - want := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.NodeAgentName, - Labels: labels, - }, - RoleRef: wantRoleRef, - Subjects: wantSubjects, - } - - existing, err := kubeClient.RbacV1().ClusterRoleBindings().Get(ctx, monov1alpha1.NodeAgentName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, want, metav1.CreateOptions{}) - return err - } - if err != nil { - return err - } - - // roleRef is immutable. If it differs, fail loudly instead of pretending we can patch it. - if !reflect.DeepEqual(existing.RoleRef, want.RoleRef) { - return fmt.Errorf("existing ClusterRoleBinding %q has different roleRef and must be recreated", monov1alpha1.NodeAgentName) - } - - changed := false - if !reflect.DeepEqual(existing.Labels, want.Labels) { - existing.Labels = want.Labels - changed = true - } - if !reflect.DeepEqual(existing.Subjects, want.Subjects) { - existing.Subjects = want.Subjects - changed = true - } - - if !changed { - return nil - } - - _, err = kubeClient.RbacV1().ClusterRoleBindings().Update(ctx, existing, metav1.UpdateOptions{}) - return err -} - -func applyNodeAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Interface, namespace string, labels map[string]string) error { - privileged := true - - dsLabels := monov1alpha1.NodeAgentLabels() - - want := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: monov1alpha1.NodeAgentName, - Namespace: namespace, - Labels: labels, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": monov1alpha1.NodeAgentName, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: dsLabels, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: monov1alpha1.NodeAgentName, - HostNetwork: true, - HostPID: true, - DNSPolicy: corev1.DNSClusterFirstWithHostNet, - NodeSelector: map[string]string{ - monov1alpha1.NodeControlKey: "true", - }, - Tolerations: []corev1.Toleration{ - {Operator: corev1.TolerationOpExists}, - }, - Containers: []corev1.Container{ - { - Name: "agent", - Image: controlAgentImage, - ImagePullPolicy: corev1.PullNever, - Args: []string{"agent", "--env-file", "$(CLUSTER_ENV_FILE)"}, - Env: []corev1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "CLUSTER_ENV_FILE", - Value: "/host/opt/monok8s/config/cluster.env", - }, - { - Name: "FW_ENV_CONFIG_FILE", - Value: "/host/etc/fw_env.config", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "host-dev", - MountPath: "/dev", - }, - { - Name: "host-etc", - MountPath: "/host/etc", - ReadOnly: true, - }, - { - Name: "host-config", - MountPath: "/host/opt/monok8s/config", - ReadOnly: true, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "host-dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - Type: hostPathType(corev1.HostPathDirectory), - }, - }, - }, - { - Name: "host-etc", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc", - Type: hostPathType(corev1.HostPathDirectory), - }, - }, - }, - { - Name: "host-config", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/opt/monok8s/config", - Type: hostPathType(corev1.HostPathDirectory), - }, - }, - }, - }, - }, - }, - }, - } - - existing, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, monov1alpha1.NodeAgentName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = kubeClient.AppsV1().DaemonSets(namespace).Create(ctx, want, metav1.CreateOptions{}) - return err - } - if err != nil { - return err - } - - changed := false - if !reflect.DeepEqual(existing.Labels, want.Labels) { - existing.Labels = want.Labels - changed = true - } - if !reflect.DeepEqual(existing.Spec, want.Spec) { - existing.Spec = want.Spec - changed = true - } - - if !changed { - return nil - } - - _, err = kubeClient.AppsV1().DaemonSets(namespace).Update(ctx, existing, metav1.UpdateOptions{}) - return err -} - -func hostPathType(t corev1.HostPathType) *corev1.HostPathType { - return &t -} - -func mountPropagationMode(m corev1.MountPropagationMode) *corev1.MountPropagationMode { - return &m -} diff --git a/clitools/pkg/node/kubeadm.go b/clitools/pkg/node/kubeadm.go index 07d04e3..1346aa5 100644 --- a/clitools/pkg/node/kubeadm.go +++ b/clitools/pkg/node/kubeadm.go @@ -12,9 +12,6 @@ import ( "time" "gopkg.in/yaml.v3" - "k8s.io/client-go/discovery" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" @@ -27,6 +24,16 @@ const ( tmpKubeadmInitConf = "/tmp/kubeadm-init.yaml" ) +func chooseVersionKubeconfig(state *LocalClusterState) string { + if state.HasAdminKubeconfig { + return adminKubeconfigPath + } + if state.HasKubeletKubeconfig { + return kubeletKubeconfigPath + } + return "" +} + func DetectLocalClusterState(ctx context.Context, nctx *NodeContext) error { _ = ctx @@ -259,110 +266,6 @@ func waitForAPIViaKubeconfig(ctx context.Context, kubeconfigPath string, timeout } } -func getServerVersion(ctx context.Context, kubeconfigPath string) (string, error) { - restCfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) - if err != nil { - return "", fmt.Errorf("build kubeconfig %s: %w", kubeconfigPath, err) - } - - // Keep this short. This is a probe, not a long-running client. - restCfg.Timeout = 5 * time.Second - - clientset, err := kubernetes.NewForConfig(restCfg) - if err != nil { - return "", fmt.Errorf("create clientset: %w", err) - } - - disc := clientset.Discovery() - return discoverServerVersion(ctx, disc) -} - -func discoverServerVersion(ctx context.Context, disc discovery.DiscoveryInterface) (string, error) { - info, err := disc.ServerVersion() - if err != nil { - return "", err - } - if info == nil || strings.TrimSpace(info.GitVersion) == "" { - return "", errors.New("server version is empty") - } - return normalizeKubeVersion(info.GitVersion), nil -} - -type kubeVersion struct { - Major int - Minor int - Patch int -} - -func parseKubeVersion(s string) (kubeVersion, error) { - s = strings.TrimSpace(s) - s = strings.TrimPrefix(s, "v") - - var v kubeVersion - n, err := fmt.Sscanf(s, "%d.%d.%d", &v.Major, &v.Minor, &v.Patch) - // Accepts "1.29" or "1.29.3" - if err != nil || n < 2 { - return kubeVersion{}, fmt.Errorf("invalid kubernetes version %q", s) - } - return v, nil -} - -// Control-plane: keep this strict. -// Accept same version, or a one-minor step where the node binary is newer than the current cluster. -// That covers normal control-plane upgrade flow but blocks nonsense. -func isSupportedControlPlaneSkew(clusterVersion, nodeVersion string) bool { - cv, err := parseKubeVersion(clusterVersion) - if err != nil { - return false - } - nv, err := parseKubeVersion(nodeVersion) - if err != nil { - return false - } - - if cv.Major != nv.Major { - return false - } - if cv.Minor == nv.Minor { - return true - } - if nv.Minor == cv.Minor+1 { - return true - } - return false -} - -// Worker: kubelet generally must not be newer than the apiserver. -// Older kubelets are allowed within supported skew range. -// Your requirement says unsupported worker skew should still proceed, so this -// only classifies support status and must NOT be used to block this function. -func isSupportedWorkerSkew(clusterVersion, nodeVersion string) bool { - cv, err := parseKubeVersion(clusterVersion) - if err != nil { - return false - } - nv, err := parseKubeVersion(nodeVersion) - if err != nil { - return false - } - - if cv.Major != nv.Major { - return false - } - - // kubelet newer than apiserver => unsupported - if nv.Minor > cv.Minor { - return false - } - - // kubelet up to 3 minors older than apiserver => supported - if cv.Minor-nv.Minor <= 3 { - return true - } - - return false -} - func ValidateRequiredImagesPresent(ctx context.Context, n *NodeContext) error { if n.Config.Spec.SkipImageCheck { klog.Infof("skipping image check (skipImageCheck=true)") @@ -419,31 +322,6 @@ func checkImagePresent(ctx context.Context, n *NodeContext, image string) error return nil } -func chooseVersionKubeconfig(state *LocalClusterState) string { - if state.HasAdminKubeconfig { - return adminKubeconfigPath - } - if state.HasKubeletKubeconfig { - return kubeletKubeconfigPath - } - return "" -} - -func versionEq(a, b string) bool { - return normalizeKubeVersion(a) == normalizeKubeVersion(b) -} - -func normalizeKubeVersion(v string) string { - v = strings.TrimSpace(v) - if v == "" { - return "" - } - if !strings.HasPrefix(v, "v") { - v = "v" + v - } - return v -} - func buildNodeRegistration(spec monov1alpha1.MonoKSConfigSpec) NodeRegistrationOptions { nodeName := strings.TrimSpace(spec.NodeName) criSocket := strings.TrimSpace(spec.ContainerRuntimeEndpoint) @@ -781,11 +659,6 @@ func RunKubeadmJoin(ctx context.Context, nctx *NodeContext) error { return nil } -func RunKubeadmUpgradeNode(context.Context, *NodeContext) error { - klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node") - return nil -} - func ReconcileControlPlane(ctx context.Context, nctx *NodeContext) error { if nctx.BootstrapState == nil { return errors.New("BootstrapState is nil, call ClassifyBootstrapAction() first") diff --git a/clitools/pkg/node/kubeadm_upgrade.go b/clitools/pkg/node/kubeadm_upgrade.go index ade55e1..c00ad25 100644 --- a/clitools/pkg/node/kubeadm_upgrade.go +++ b/clitools/pkg/node/kubeadm_upgrade.go @@ -257,3 +257,95 @@ func describeHealthCheckFailure(ctx context.Context, kubeClient kubernetes.Inter return nil } + +func RunKubeadmUpgradeNode(ctx context.Context, nctx *NodeContext) error { + if nctx == nil { + return errors.New("node context is nil") + } + if nctx.Config == nil { + return errors.New("node config is nil") + } + if nctx.LocalClusterState == nil { + return errors.New("LocalClusterState is nil. Please run earlier steps first") + } + if nctx.BootstrapState == nil { + return errors.New("BootstrapState is nil. Please run earlier steps first") + } + + switch nctx.BootstrapState.Action { + case BootstrapActionUpgradeWorker: + // continue + default: + klog.V(4).Infof("RunKubeadmUpgradeNode skipped for action %q", nctx.BootstrapState.Action) + return nil + } + + wantVersion := normalizeKubeVersion(strings.TrimSpace(nctx.Config.Spec.KubernetesVersion)) + if wantVersion == "" { + return errors.New("spec.kubernetesVersion is required") + } + + kubeconfigPath := chooseVersionKubeconfig(nctx.LocalClusterState) + if kubeconfigPath == "" { + return errors.New("no kubeconfig available for detecting cluster version") + } + + clusterVersion := strings.TrimSpace(nctx.BootstrapState.DetectedClusterVersion) + if clusterVersion == "" { + var err error + clusterVersion, err = getServerVersion(ctx, kubeconfigPath) + if err != nil { + if nctx.BootstrapState.UnsupportedWorkerVersionSkew { + klog.Warningf( + "cluster version unavailable but worker skew was marked unsupported/permissive, continuing: reason=%s", + nctx.BootstrapState.VersionSkewReason, + ) + } else { + return fmt.Errorf("get cluster version via %s: %w", kubeconfigPath, err) + } + } + } + + if clusterVersion != "" && !isSupportedWorkerSkew(clusterVersion, wantVersion) { + klog.Warningf( + "unsupported worker version skew detected, continuing anyway: cluster=%s node=%s", + clusterVersion, + wantVersion, + ) + } + + klog.Infof( + "running kubeadm upgrade node: role=%s clusterVersion=%s nodeVersion=%s kubeconfig=%s", + strings.TrimSpace(nctx.Config.Spec.ClusterRole), + clusterVersion, + wantVersion, + kubeconfigPath, + ) + + args := []string{ + "upgrade", + "node", + "--kubeconfig", + kubeconfigPath, + } + + _, err := nctx.SystemRunner.RunWithOptions( + ctx, + "kubeadm", + args, + system.RunOptions{ + Timeout: 10 * time.Minute, + OnStdoutLine: func(line string) { + klog.Infof("[kubeadm] %s", line) + }, + OnStderrLine: func(line string) { + klog.Infof("[kubeadm] %s", line) + }, + }, + ) + if err != nil { + return fmt.Errorf("run kubeadm upgrade node: %w", err) + } + + return nil +} diff --git a/clitools/pkg/node/prereqs.go b/clitools/pkg/node/prereqs.go index 6551116..030c1a8 100644 --- a/clitools/pkg/node/prereqs.go +++ b/clitools/pkg/node/prereqs.go @@ -8,9 +8,18 @@ import ( "strings" "time" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" ) +type kubeVersion struct { + Major int + Minor int + Patch int +} + func ValidateNodeIPAndAPIServerReachability(ctx context.Context, nct *NodeContext) error { requireLocalIP := func(wantedIP string) error { wantedIP = strings.TrimSpace(wantedIP) @@ -189,3 +198,116 @@ func CheckForVersionSkew(ctx context.Context, nctx *NodeContext) error { return nil } + +func versionEq(a, b string) bool { + return normalizeKubeVersion(a) == normalizeKubeVersion(b) +} + +func normalizeKubeVersion(v string) string { + v = strings.TrimSpace(v) + if v == "" { + return "" + } + if !strings.HasPrefix(v, "v") { + v = "v" + v + } + return v +} + +func parseKubeVersion(s string) (kubeVersion, error) { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "v") + + var v kubeVersion + n, err := fmt.Sscanf(s, "%d.%d.%d", &v.Major, &v.Minor, &v.Patch) + // Accepts "1.29" or "1.29.3" + if err != nil || n < 2 { + return kubeVersion{}, fmt.Errorf("invalid kubernetes version %q", s) + } + return v, nil +} + +// Control-plane: keep this strict. +// Accept same version, or a one-minor step where the node binary is newer than the current cluster. +// That covers normal control-plane upgrade flow but blocks nonsense. +func isSupportedControlPlaneSkew(clusterVersion, nodeVersion string) bool { + cv, err := parseKubeVersion(clusterVersion) + if err != nil { + return false + } + nv, err := parseKubeVersion(nodeVersion) + if err != nil { + return false + } + + if cv.Major != nv.Major { + return false + } + if cv.Minor == nv.Minor { + return true + } + if nv.Minor == cv.Minor+1 { + return true + } + return false +} + +// Worker: kubelet generally must not be newer than the apiserver. +// Older kubelets are allowed within supported skew range. +// Your requirement says unsupported worker skew should still proceed, so this +// only classifies support status and must NOT be used to block this function. +func isSupportedWorkerSkew(clusterVersion, nodeVersion string) bool { + cv, err := parseKubeVersion(clusterVersion) + if err != nil { + return false + } + nv, err := parseKubeVersion(nodeVersion) + if err != nil { + return false + } + + if cv.Major != nv.Major { + return false + } + + // kubelet newer than apiserver => unsupported + if nv.Minor > cv.Minor { + return false + } + + // kubelet up to 3 minors older than apiserver => supported + if cv.Minor-nv.Minor <= 3 { + return true + } + + return false +} + +func getServerVersion(ctx context.Context, kubeconfigPath string) (string, error) { + restCfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return "", fmt.Errorf("build kubeconfig %s: %w", kubeconfigPath, err) + } + + // Keep this short. This is a probe, not a long-running client. + restCfg.Timeout = 5 * time.Second + + clientset, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return "", fmt.Errorf("create clientset: %w", err) + } + + disc := clientset.Discovery() + return discoverServerVersion(ctx, disc) +} + +func discoverServerVersion(ctx context.Context, disc discovery.DiscoveryInterface) (string, error) { + info, err := disc.ServerVersion() + if err != nil { + return "", err + } + if info == nil || strings.TrimSpace(info.GitVersion) == "" { + return "", errors.New("server version is empty") + } + return normalizeKubeVersion(info.GitVersion), nil +} diff --git a/clitools/pkg/render/agent.go b/clitools/pkg/render/agent.go new file mode 100644 index 0000000..185d3b1 --- /dev/null +++ b/clitools/pkg/render/agent.go @@ -0,0 +1,284 @@ +package render + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" + buildinfo "example.com/monok8s/pkg/buildinfo" +) + +type AgentConf struct { + Namespace string + Image string + ImagePullSecrets []string + Labels map[string]string +} + +func RenderAgentDaemonSets(conf AgentConf) (string, error) { + objs, err := buildAgentDaemonSetObjects(conf) + if err != nil { + return "", err + } + + return renderObjects(objs) +} + +func buildAgentDaemonSetObjects(conf AgentConf) ([]runtime.Object, error) { + if strings.TrimSpace(conf.Namespace) == "" { + return nil, fmt.Errorf("namespace is required") + } + + conf.Labels = map[string]string{ + "app.kubernetes.io/name": monov1alpha1.NodeAgentName, + "app.kubernetes.io/component": "agent", + "app.kubernetes.io/part-of": "monok8s", + "app.kubernetes.io/managed-by": monov1alpha1.NodeControlName, + } + + return []runtime.Object{ + buildAgentServiceAccount(conf), + buildAgentClusterRole(conf), + buildAgentClusterRoleBinding(conf), + buildAgentDaemonSet(conf), + }, nil +} + +func buildAgentNamespace(conf AgentConf) *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: conf.Namespace, + Labels: copyStringMap(conf.Labels), + }, + } +} + +func buildAgentServiceAccount(conf AgentConf) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: monov1alpha1.NodeAgentName, + Namespace: conf.Namespace, + Labels: copyStringMap(conf.Labels), + }, + } +} + +func buildAgentClusterRole(conf AgentConf) *rbacv1.ClusterRole { + wantRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{monov1alpha1.Group}, + Resources: []string{"osupgrades"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{monov1alpha1.Group}, + Resources: []string{"osupgradeprogresses"}, + Verbs: []string{"get", "list", "watch", "create", "patch", "update"}, + }, + { + APIGroups: []string{monov1alpha1.Group}, + Resources: []string{"osupgradeprogresses/status"}, + Verbs: []string{"get", "list", "watch", "create", "patch", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: monov1alpha1.NodeAgentName, + Labels: copyStringMap(conf.Labels), + }, + Rules: wantRules, + } +} + +func buildAgentClusterRoleBinding(conf AgentConf) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: monov1alpha1.NodeAgentName, + Labels: copyStringMap(conf.Labels), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: monov1alpha1.NodeAgentName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: monov1alpha1.NodeAgentName, + Namespace: conf.Namespace, + }, + }, + } +} + +func buildAgentDaemonSet(conf AgentConf) *appsv1.DaemonSet { + privileged := true + dsLabels := monov1alpha1.NodeAgentLabels() + + image, pullPolicy := agentImage(conf) + + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: monov1alpha1.NodeAgentName, + Namespace: conf.Namespace, + Labels: copyStringMap(conf.Labels), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": monov1alpha1.NodeAgentName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: dsLabels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: monov1alpha1.NodeAgentName, + HostNetwork: true, + HostPID: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + ImagePullSecrets: imagePullSecrets(conf.ImagePullSecrets), + NodeSelector: map[string]string{ + monov1alpha1.NodeControlKey: "true", + }, + Tolerations: []corev1.Toleration{ + {Operator: corev1.TolerationOpExists}, + }, + Containers: []corev1.Container{ + { + Name: "agent", + Image: image, + ImagePullPolicy: pullPolicy, + Args: []string{"agent", "--env-file", "$(CLUSTER_ENV_FILE)"}, + Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "CLUSTER_ENV_FILE", + Value: "/host/opt/monok8s/config/cluster.env", + }, + { + Name: "FW_ENV_CONFIG_FILE", + Value: "/host/etc/fw_env.config", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host-dev", + MountPath: "/dev", + }, + { + Name: "host-etc", + MountPath: "/host/etc", + ReadOnly: true, + }, + { + Name: "host-config", + MountPath: "/host/opt/monok8s/config", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "host-dev", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev", + Type: hostPathType(corev1.HostPathDirectory), + }, + }, + }, + { + Name: "host-etc", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc", + Type: hostPathType(corev1.HostPathDirectory), + }, + }, + }, + { + Name: "host-config", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/opt/monok8s/config", + Type: hostPathType(corev1.HostPathDirectory), + }, + }, + }, + }, + }, + }, + }, + } +} + +func agentImage(conf AgentConf) (string, corev1.PullPolicy) { + if conf.Image != "" { + return conf.Image, corev1.PullIfNotPresent + } + + return fmt.Sprintf("localhost/monok8s/node-control:%s", buildinfo.Version), corev1.PullNever +} + +func copyStringMap(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + + return out +} + +func hostPathType(t corev1.HostPathType) *corev1.HostPathType { + return &t +} diff --git a/clitools/pkg/render/agent_apply.go b/clitools/pkg/render/agent_apply.go new file mode 100644 index 0000000..f24a19c --- /dev/null +++ b/clitools/pkg/render/agent_apply.go @@ -0,0 +1,203 @@ +package render + +import ( + "context" + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" +) + +func ApplyAgentDaemonSets(ctx context.Context, kubeClient kubernetes.Interface, conf AgentConf) error { + objs, err := buildAgentDaemonSetObjects(conf) + if err != nil { + return err + } + + if err := applyAgentNamespace(ctx, kubeClient, buildAgentNamespace(conf)); err != nil { + return fmt.Errorf("apply namespace: %w", err) + } + + for _, obj := range objs { + if err := applyAgentObject(ctx, kubeClient, obj); err != nil { + return err + } + } + + return nil +} + +func applyAgentObject(ctx context.Context, kubeClient kubernetes.Interface, obj runtime.Object) error { + switch want := obj.(type) { + case *corev1.ServiceAccount: + return applyAgentServiceAccount(ctx, kubeClient, want) + case *rbacv1.ClusterRole: + return applyAgentClusterRole(ctx, kubeClient, want) + case *rbacv1.ClusterRoleBinding: + return applyAgentClusterRoleBinding(ctx, kubeClient, want) + case *appsv1.DaemonSet: + return applyAgentDaemonSet(ctx, kubeClient, want) + default: + return fmt.Errorf("unsupported agent object type %T", obj) + } +} + +func applyAgentNamespace(ctx context.Context, kubeClient kubernetes.Interface, want *corev1.Namespace) error { + existing, err := kubeClient.CoreV1().Namespaces().Get(ctx, want.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = kubeClient.CoreV1().Namespaces().Create(ctx, want, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + + labels, changed := mergeStringMapsInto(existing.Labels, want.Labels) + if !changed { + return nil + } + + existing.Labels = labels + _, err = kubeClient.CoreV1().Namespaces().Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func applyAgentServiceAccount(ctx context.Context, kubeClient kubernetes.Interface, want *corev1.ServiceAccount) error { + existing, err := kubeClient.CoreV1().ServiceAccounts(want.Namespace).Get(ctx, want.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = kubeClient.CoreV1().ServiceAccounts(want.Namespace).Create(ctx, want, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + + changed := false + if !reflect.DeepEqual(existing.Labels, want.Labels) { + existing.Labels = want.Labels + changed = true + } + + if !changed { + return nil + } + + _, err = kubeClient.CoreV1().ServiceAccounts(want.Namespace).Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func applyAgentClusterRole(ctx context.Context, kubeClient kubernetes.Interface, want *rbacv1.ClusterRole) error { + existing, err := kubeClient.RbacV1().ClusterRoles().Get(ctx, want.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = kubeClient.RbacV1().ClusterRoles().Create(ctx, want, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + + changed := false + if !reflect.DeepEqual(existing.Labels, want.Labels) { + existing.Labels = want.Labels + changed = true + } + if !reflect.DeepEqual(existing.Rules, want.Rules) { + existing.Rules = want.Rules + changed = true + } + + if !changed { + return nil + } + + _, err = kubeClient.RbacV1().ClusterRoles().Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func applyAgentClusterRoleBinding(ctx context.Context, kubeClient kubernetes.Interface, want *rbacv1.ClusterRoleBinding) error { + existing, err := kubeClient.RbacV1().ClusterRoleBindings().Get(ctx, want.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, want, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + + // roleRef is immutable. If it differs, fail loudly instead of pretending we can patch it. + if !reflect.DeepEqual(existing.RoleRef, want.RoleRef) { + return fmt.Errorf("existing ClusterRoleBinding %q has different roleRef and must be recreated", want.Name) + } + + changed := false + if !reflect.DeepEqual(existing.Labels, want.Labels) { + existing.Labels = want.Labels + changed = true + } + if !reflect.DeepEqual(existing.Subjects, want.Subjects) { + existing.Subjects = want.Subjects + changed = true + } + + if !changed { + return nil + } + + _, err = kubeClient.RbacV1().ClusterRoleBindings().Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func applyAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Interface, want *appsv1.DaemonSet) error { + existing, err := kubeClient.AppsV1().DaemonSets(want.Namespace).Get(ctx, want.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = kubeClient.AppsV1().DaemonSets(want.Namespace).Create(ctx, want, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + + changed := false + if !reflect.DeepEqual(existing.Labels, want.Labels) { + existing.Labels = want.Labels + changed = true + } + if !reflect.DeepEqual(existing.Spec, want.Spec) { + existing.Spec = want.Spec + changed = true + } + + if !changed { + return nil + } + + _, err = kubeClient.AppsV1().DaemonSets(want.Namespace).Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func mergeStringMapsInto(dst map[string]string, src map[string]string) (map[string]string, bool) { + if len(src) == 0 { + return dst, false + } + + changed := false + if dst == nil { + dst = map[string]string{} + changed = true + } + + for k, v := range src { + if dst[k] != v { + dst[k] = v + changed = true + } + } + + return dst, changed +} diff --git a/clitools/pkg/render/controller.go b/clitools/pkg/render/controller.go index c92e429..ee8eb8b 100644 --- a/clitools/pkg/render/controller.go +++ b/clitools/pkg/render/controller.go @@ -1,7 +1,6 @@ package render import ( - "bytes" "fmt" appsv1 "k8s.io/api/apps/v1" @@ -9,7 +8,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/intstr" monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" @@ -17,9 +15,10 @@ import ( ) type ControllerConf struct { - Namespace string - Image string - Labels map[string]string + Namespace string + Image string + ImagePullSecrets []string + Labels map[string]string } func RenderControllerDeployments(conf ControllerConf) (string, error) { @@ -41,27 +40,7 @@ func RenderControllerDeployments(conf ControllerConf) (string, error) { buildControllerDeployment(conf), } - s := runtime.NewScheme() - _ = corev1.AddToScheme(s) - _ = rbacv1.AddToScheme(s) - _ = appsv1.AddToScheme(s) - - serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, s, s) - - var buf bytes.Buffer - - for i, obj := range objs { - if i > 0 { - if _, err := fmt.Fprintln(&buf, "---"); err != nil { - return "", err - } - } - if err := serializer.Encode(obj, &buf); err != nil { - return "", err - } - } - - return buf.String(), nil + return renderObjects(objs) } func buildControllerServiceAccount(conf ControllerConf) *corev1.ServiceAccount { @@ -191,6 +170,7 @@ func buildControllerDeployment(conf ControllerConf) *appsv1.Deployment { }, Spec: corev1.PodSpec{ ServiceAccountName: monov1alpha1.ControllerName, + ImagePullSecrets: imagePullSecrets(conf.ImagePullSecrets), Containers: []corev1.Container{ { Name: "controller", diff --git a/clitools/pkg/render/helpers.go b/clitools/pkg/render/helpers.go new file mode 100644 index 0000000..1d8bcea --- /dev/null +++ b/clitools/pkg/render/helpers.go @@ -0,0 +1,74 @@ +package render + +import ( + "bytes" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" +) + +func renderObjects(objs []runtime.Object) (string, error) { + var buf bytes.Buffer + + for i, obj := range objs { + if i > 0 { + if _, err := fmt.Fprintln(&buf, "---"); err != nil { + return "", err + } + } + + b, err := renderObjectYAML(obj) + if err != nil { + return "", err + } + + if _, err := buf.Write(b); err != nil { + return "", err + } + } + + return buf.String(), nil +} + +func renderObjectYAML(obj runtime.Object) ([]byte, error) { + b, err := yaml.Marshal(obj) + if err != nil { + return nil, err + } + + var m map[string]any + if err := yaml.Unmarshal(b, &m); err != nil { + return nil, err + } + + delete(m, "status") + + return yaml.Marshal(m) +} + +func imagePullSecrets(names []string) []corev1.LocalObjectReference { + if len(names) == 0 { + return nil + } + + refs := make([]corev1.LocalObjectReference, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + continue + } + + refs = append(refs, corev1.LocalObjectReference{ + Name: name, + }) + } + + if len(refs) == 0 { + return nil + } + + return refs +} diff --git a/clitools/pkg/render/sshd.go b/clitools/pkg/render/sshd.go index 94b6799..735385c 100644 --- a/clitools/pkg/render/sshd.go +++ b/clitools/pkg/render/sshd.go @@ -1,16 +1,11 @@ package render import ( - "bytes" - "fmt" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/intstr" monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1" @@ -39,27 +34,7 @@ func RenderSSHDDeployments(namespace, authKeys string) (string, error) { buildSSHDDeployment(vals, namespace, labels), } - s := runtime.NewScheme() - _ = corev1.AddToScheme(s) - _ = rbacv1.AddToScheme(s) - _ = appsv1.AddToScheme(s) - - serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, s, s) - - var buf bytes.Buffer - - for i, obj := range objs { - if i > 0 { - if _, err := fmt.Fprintln(&buf, "---"); err != nil { - return "", err - } - } - if err := serializer.Encode(obj, &buf); err != nil { - return "", err - } - } - - return buf.String(), nil + return renderObjects(objs) } func buildSSHDConfigMap(