diff --git a/.envrc.example b/.envrc.example index e464c745..1d236559 100644 --- a/.envrc.example +++ b/.envrc.example @@ -2,6 +2,8 @@ export KUBECONFIG=$PWD/.mgt-cluster-kubeconfig.yaml export PATH=$PWD/hack/tools/bin:$PATH +# Legacy variables from the previous workflow. They are currently not consumed by hack scripts, but are kept here for compatibility/reference. + export PROVIDER=docker # Versions @@ -21,3 +23,18 @@ export DISABLE_VERSIONCHECK="true" # Release export RELEASE_CLUSTER_CLASS=$CLUSTER_CLASS_NAME export RELEASE_KUBERNETES_VERSION=1-27 + +# Current workflow: the build scripts derive the stack from PROVIDER and + +export PROVIDER=openstack +export CLUSTER_STACK=scs + +# OCI registry (leave unset for auto-configured ttl.sh during development) +# export OCI_REGISTRY=registry.example.com +# export OCI_REPOSITORY=kaas/cluster-stacks +# export OCI_USERNAME=user +# export OCI_PASSWORD=password +# export OCI_ACCESS_TOKEN=token + +# Optional: GitHub token for higher API rate limits during version resolution +# export GITHUB_TOKEN=ghp_... \ No newline at end of file diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..4abb6d29 --- /dev/null +++ b/Containerfile @@ -0,0 +1,53 @@ +FROM alpine:3.21 + +LABEL org.opencontainers.image.source="https://github.com/SovereignCloudStack/cluster-stacks" +LABEL org.opencontainers.image.description="Cluster Stack Build Tools" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +# Install system dependencies +RUN apk add --no-cache \ + bash \ + git \ + curl \ + tar \ + gzip \ + gawk \ + python3 \ + py3-yaml \ + jq \ + ca-certificates + +# Install helm +RUN HELM_VERSION=v3.17.3 && \ + curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | \ + tar -xz -C /usr/local/bin --strip-components=1 linux-amd64/helm + +# Install yq (mikefarah) +RUN YQ_VERSION=v4.45.4 && \ + curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + +# Install oras +RUN ORAS_VERSION=1.2.2 && \ + curl -fsSL "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" | \ + tar -xz -C /usr/local/bin oras + +# Install just +RUN JUST_VERSION=1.40.0 && \ + curl -fsSL "https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" | \ + tar -xz -C /usr/local/bin just + +WORKDIR /workspace + +# Verify installations +RUN bash --version | head -1 && \ + helm version --short && \ + yq --version && \ + oras version && \ + just --version && \ + git --version + +# Allow git operations inside mounted volumes +RUN git config --global --add safe.directory /workspace + +CMD ["/bin/bash"] diff --git a/Makefile b/Makefile deleted file mode 100644 index e5d63845..00000000 --- a/Makefile +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec -.DEFAULT_GOAL:=help - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: ensure-connected-to-mgt-cluster -ensure-connected-to-mgt-cluster: - ./hack/ensure-connected-to-mgt-cluster.sh - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -############# -# Variables # -############# - -TIMEOUT := $(shell command -v timeout || command -v gtimeout) - -# Directories -BIN_DIR := bin -TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) -export GOBIN := $(abspath $(TOOLS_BIN_DIR)) - -ARTIFACTS_PATH := $(ROOT_DIR)/_artifacts -# Docker -RM := --rm -TTY := -t - -##@ Binaries -############ -# Binaries # -############ - -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize) -kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize -$(KUSTOMIZE): # Build kustomize from tools folder. - go install sigs.k8s.io/kustomize/kustomize/v5@v5.1.0 - -ENVSUBST := $(abspath $(TOOLS_BIN_DIR)/envsubst) -envsubst: $(ENVSUBST) ## Build a local copy of envsubst -$(ENVSUBST): # Build envsubst from tools folder. - go install github.com/drone/envsubst/v2/cmd/envsubst@latest - -CTLPTL := $(abspath $(TOOLS_BIN_DIR)/ctlptl) -ctlptl: $(CTLPTL) ## Build a local copy of ctlptl -$(CTLPTL): - go install github.com/tilt-dev/ctlptl/cmd/ctlptl@v0.8.20 - -CLUSTERCTL := $(abspath $(TOOLS_BIN_DIR)/clusterctl) -clusterctl: $(CLUSTERCTL) ## Build a local copy of clusterctl -$(CLUSTERCTL): - curl -sSLf https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.1/clusterctl-linux-amd64 -o $(CLUSTERCTL) - chmod a+rx $(CLUSTERCTL) - -HELM := $(abspath $(TOOLS_BIN_DIR)/helm) -helm: $(HELM) ## Build a local copy of helm -$(HELM): - go install helm.sh/helm/v3/cmd/helm@v3.12.3 - -KIND := $(abspath $(TOOLS_BIN_DIR)/kind) -kind: $(KIND) ## Build a local copy of kind -$(KIND): - go install sigs.k8s.io/kind@v0.20.0 - -all-tools: $(KIND) $(CTLPTL) $(KIND) $(ENVSUBST) $(KUSTOMIZE) $(CLUSTERCTL) $(HELM) - echo 'done' - -.PHONY: basics -basics: $(KIND) $(CTLPTL) $(KIND) $(ENVSUBST) $(KUSTOMIZE) $(CLUSTERCTL) - @./hack/ensure-env-variables.sh CAPI_VERSION CAPD_VERSION NAMESPACE \ - CLUSTER_CLASS_NAME K8S_VERSION CLUSTER_NAME PROVIDER - @mkdir -p build - -##@ Development -############### -# Development # -############### - -.PHONY: cluster -kind-cluster: basics ## Creates kind-dev Cluster - ./hack/kind-dev.sh - kubectl config set-context --current --namespace $(NAMESPACE) - -.PHONY: watch -watch: ## Show the current state of the CRDs and events. - watch -c "kubectl -n $(NAMESPACE) get cluster; echo; kubectl -n $(NAMESPACE) get machine; echo; kubectl -n $(NAMESPACE) get dockermachine; echo; echo Events; kubectl -A get events --sort-by=metadata.creationTimestamp | tail -5" - -##@ Clean -######### -# Clean # -######### -.PHONY: clean -clean: basics ## Remove all generated files - $(MAKE) clean-bin - -.PHONY: clean-bin -clean-bin: ## Remove all generated helper binaries - rm -rf $(TOOLS_BIN_DIR) - -.PHONY: clean-release -clean-release: ## Remove the release folder - rm -rf $(RELEASE_DIR) - -.PHONY: clean-release-git -clean-release-git: ## Restores the git files usually modified during a release - git restore ./*manager_config_patch.yaml ./*manager_pull_policy.yaml - -##@ Releasing -############# -# Releasing # -############# -## latest git tag for the commit, e.g., v0.3.10 -RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null) -# the previous release tag, e.g., v0.3.9, excluding pre-release tags -PREVIOUS_TAG ?= $(shell git tag -l | grep -E "^v[0-9]+\.[0-9]+\.[0-9]." | sort -V | grep -B1 $(RELEASE_TAG) | head -n 1 2>/dev/null) -RELEASE_DIR ?= out -RELEASE_NOTES_DIR := _releasenotes - -$(RELEASE_DIR): - mkdir -p $(RELEASE_DIR)/ - -$(RELEASE_NOTES_DIR): - mkdir -p $(RELEASE_NOTES_DIR)/ - -.PHONY: test-release -test-release: basics - $(MAKE) set-manifest-image MANIFEST_IMG=$(IMAGE_PREFIX)/capd-staging MANIFEST_TAG=$(TAG) - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent - $(MAKE) release-manifests - -.PHONY: release-manifests -release-manifests: basics generate-manifests generate-go-deepcopy $(RELEASE_DIR) cluster-templates ## Builds the manifests to publish with a release - $(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml - ## Build capd-components (aggregate of all of the above). - cp metadata.yaml $(RELEASE_DIR)/metadata.yaml - cp templates/cluster-templates/cluster-template* $(RELEASE_DIR)/ - cp templates/cluster-templates/cluster-class* $(RELEASE_DIR)/ - -.PHONY: release-notes -release-notes: basics $(RELEASE_NOTES_DIR) $(RELEASE_NOTES) - go run ./hack/tools/release/notes.go --from=$(PREVIOUS_TAG) > $(RELEASE_NOTES_DIR)/$(RELEASE_TAG).md - - -##@ Testing -########### -# Testing # -########### -ARTIFACTS ?= _artifacts -$(ARTIFACTS): - mkdir -p $(ARTIFACTS)/ - -##@ cluster-class -################# -# cluster-class # -################# - -.PHONY: re-apply-cluster-class-docker -re-apply-cluster-class-docker: ensure-connected-to-mgt-cluster basics ## Re-apply only a cluster-class. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class | kubectl -n $(NAMESPACE) apply -f - - -.PHONY: delete-cluster-class-docker -delete-cluster-class-docker: ensure-connected-to-mgt-cluster basics ## Delete a cluster-class. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class | kubectl -n $(NAMESPACE) delete -f - - -##@ cluster-addon -################# -# cluster-addon # -################# - -## working on cluster-addon -.PHONY: generate-deps-cluster-addon-docker -generate-deps-cluster-addon-docker: ensure-connected-to-mgt-cluster basics ## Build a cluster-class. - cd providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon && rm -rf ./charts/* && helm dependency update - -.PHONY: package-cluster-addon-docker -package-cluster-addon-docker: ensure-connected-to-mgt-cluster basics ## Build a cluster-class. - @mkdir -p .helm - $(HELM) package providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon -d .helm - -##@ Main Targets -################ -# Main Targets # -################ -.PHONY: delete-bootstrap-cluster -delete-bootstrap-cluster: ensure-connected-to-mgt-cluster basics ## Deletes Kind-dev Cluster - $(CTLPTL) delete cluster kind-scs-cluster-stacks - $(CTLPTL) delete registry cluster-stacks-registry - -.PHONY: create-bootstrap-cluster -create-bootstrap-cluster: basics kind-cluster ## Create mgt-cluster and install capi-stack. - EXP_RUNTIME_SDK=true CLUSTER_TOPOLOGY=true DISABLE_VERSIONCHECK="true" $(CLUSTERCTL) init --core cluster-api:$(CAPI_VERSION) --bootstrap kubeadm:$(CAPI_VERSION) --control-plane kubeadm:$(CAPI_VERSION) - # kubectl apply -f https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/$(CAAPH_VERSION)/add-on-components.yaml - kubectl wait -n cert-manager deployment cert-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-kubeadm-bootstrap-system deployment capi-kubeadm-bootstrap-controller-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-kubeadm-control-plane-system deployment capi-kubeadm-control-plane-controller-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-system deployment capi-controller-manager --for=condition=Available --timeout=300s - -.PHONY: install-provider-docker -install-provider-docker: create-bootstrap-cluster ## Install Docker Infrastructure provider. - # hangs for ever waiting for cert-manger to get available if called twice. - if kubectl get deployments.apps -n capd-system capd-controller-manager > /dev/null 2>&1; then \ - echo "capd is already installed" ; \ - else \ - echo "installing capd" ; \ - DISABLE_VERSIONCHECK="true" $(CLUSTERCTL) init --infrastructure docker:$(CAPD_VERSION); \ - fi - kubectl wait -n capd-system deployment capd-controller-manager --for=condition=Available --timeout=300s - -.PHONY: prepare-provider-docker -prepare-provider-docker: install-provider-docker ## Prepares the Docker Environment. - kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f - - -.PHONY: apply-cluster-class-docker -apply-cluster-class-docker: $(HELM) prepare-provider-docker package-cluster-addon-docker ## Applies all resources and node-images. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class > build/cluster-class-created.yaml - kubectl -n $(NAMESPACE) apply -f build/cluster-class-created.yaml - # create the docker image. Start first build in background to - docker build -t docker-ferrol-1-27-controlplaneamd64-v1:dev \ - --file providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/Dockerfile.controlplane \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/ & \ - docker build -t docker-ferrol-1-27-workeramd64-v1:dev \ - --file providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/Dockerfile.worker \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/ - @echo "" - @echo "Done" - -.PHONY: create-workload-cluster -create-workload-cluster: apply-cluster-class-docker basics ## Creates a workload cluster. - cat providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/topology-$(PROVIDER).yaml | $(ENVSUBST) - > build/topology.yaml - kubectl apply -f build/topology.yaml - # Wait for the kubeconfig to become available. - ${TIMEOUT} --foreground 5m bash -c "while ! kubectl -n $(NAMESPACE) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do date; echo waiting for secret $(CLUSTER_NAME)-kubeconfig; sleep 1; done" - # Get kubeconfig and store it locally. - @mkdir -p .kubeconfigs - kubectl -n $(NAMESPACE) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig - if [ ! -s ".kubeconfigs/.$(CLUSTER_NAME)-kubeconfig" ]; then echo "failed to create .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig"; exit 1; fi - ${TIMEOUT} --foreground 15m bash -c "while ! kubectl --kubeconfig=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig -n $(NAMESPACE) get nodes | \ - grep control-plane; do echo 'Waiting for control-plane in workload-cluster'; sleep 1; done" - chmod a=,u=rw .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig - @echo "" - @echo 'Access to workload API server successful.' - @echo 'use KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig to access the workload cluster' - -.PHONY: install-cluster-addons -install-addons-in-workload-cluster: $(HELM) - # Cluster Addons are software tools which provide required functionality to the cluster. - # Example: CNI (Container Network Interface) - # Hint: Applications like PostgreSQL or Prometheus are not cluster addons. - - @# The "ugly" replacement of policy/v1beta1 needs to be done, since `helm template` - @# has no access to the cluster, and the new --dry-run=server is not released yet. - @# Replace the `sed` with `--dry-run=server` when helm supports --dry-run=server. - @# It should be included in helm 3.13.0 - KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig helm template \ - ./providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon \ - | sed 's#apiVersion: policy/v1beta1#apiVersion: policy/v1#' \ - > build/cluster-addons-$(CLUSTER_NAME).yaml - KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig kubectl apply -f build/cluster-addons-$(CLUSTER_NAME).yaml - - -.PHONY: release-docker -release-docker: clean-release $(HELM) ## Builds and push container images using the latest git tag for the commit. - # @if [ -z "${RELEASE_TAG}" ]; then echo "RELEASE_TAG is not set"; exit 1; fi - # @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi - # git checkout "${RELEASE_TAG}" - @./hack/ensure-env-variables.sh RELEASE_CLUSTER_CLASS RELEASE_KUBERNETES_VERSION - @mkdir -p .release - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-addon-values.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/node-images.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/metadata.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/topology-* .release/ - $(HELM) package providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-addon -d .release/ - $(HELM) package providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-class -d .release/ diff --git a/README.md b/README.md index 78f40263..dffebbc2 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,87 @@ # Cluster Stacks -:wave: Welcome to the SCS Cluster Stacks. The reference implementation for SCS KaaS. +Reference implementation of SCS Kubernetes-as-a-Service cluster stacks, built on +[Cluster API](https://cluster-api.sigs.k8s.io/) and managed by the +[Cluster Stack Operator](https://github.com/SovereignCloudStack/cluster-stack-operator) (CSO). -#### Useful links +## Quick start -- [Quick Start](providers/docker/scs/README.md) -- [Docs](https://docs.scs.community/docs/category/cluster-stacks) +```bash +# Prerequisites: kind, kubectl, helm, clusterctl, just +# See docs/quickstart.md for full details -## What is Cluster Stacks? +# Create a management cluster and install CAPI + provider +kind create cluster +clusterctl init --infrastructure openstack -Cluster Stacks is a comprehensive framework and set of reference -implementations for defining and managing Kubernetes clusters via the Cluster -API. Designed to support multiple providers and a broad range of Kubernetes -versions, it offers a standardized approach to configuring and operating -Kubernetes environments. For detailed information see [docs.scs.community](https://docs.scs.community/docs/container/components/cluster-stacks/components/cluster-stacks/overview). +# Install the CSO (auto-configures ttl.sh for development) +just install-cso + +# Build, publish, and generate ClusterStack resource for a specific version +just dev --version 1.35 +``` + +See [docs/quickstart.md](docs/quickstart.md) for a complete walkthrough. + +## Available cluster stacks + +| Provider | Stack | Description | Versions | +|------------|-------|------------------------------------------|----------------| +| OpenStack | scs | Standard SCS stack (dedicated VMs) | 1-32 .. 1-35 | +| OpenStack | hcp | Hosted Control Plane (CP as pods) | 1-33 .. 1-35 | +| Docker | scs | Local development stack | 1-32 .. 1-35 | + +## Repository structure + +```text +providers/ + / + / + 1-XX/ # per-Kubernetes-minor-version directory + stack.yaml # stack metadata and addon version pins + cluster-class/ # Helm chart: ClusterClass + infrastructure templates + cluster-addon/ # Helm chart: CNI, CCM, CSI, metrics-server + image-manager.yaml # OpenStack only: aggregated image references +``` + +Each `1-XX/` directory is self-contained: it carries its own `stack.yaml`, +ClusterClass definition, and addon charts. There is no shared state between +minor versions. + +## Build system + +All workflows are driven by [`just`](https://just.systems): + +```bash +just build --version 1.35 # build locally to .release/ +just publish --version 1.35 # build + push to OCI registry +just dev --version 1.35 # publish + print ClusterStack YAML +just dev --install-cso --version 1.35 # also install/upgrade CSO +just matrix # show version/addon matrix +just update versions # update Kubernetes patch versions +just update addons # update addon chart versions +just generate-resources --version 1.35 # generate ClusterStack + Cluster YAML +just generate-docs # regenerate configuration docs +``` + +Set `PROVIDER` and `CLUSTER_STACK` environment variables (or use a `.env` file) +to target a different stack (default: `openstack`/`scs`). + +## Documentation + +- [Overview](docs/overview.md) -- architecture, versioning, and structure +- [Quickstart](docs/quickstart.md) -- end-to-end guide for all providers +- [OpenStack HCP](docs/providers/openstack/hcp.md) -- Hosted Control Plane stack + +Configuration references are generated from ClusterClass definitions via +`just generate-docs`. ## Releases -Releases for the providers openstack/scs and openstack/scs2 are published in -the [SCS registry](oci://registry.scs.community/kaas/cluster-stacks). +Releases are published as OCI artifacts to the +[SCS registry](https://registry.scs.community/kaas/cluster-stacks). ## Community - [Matrix](https://matrix.to/#/!NZpJdPGjAHISXnHUil:matrix.org) -- [notes](https://input.scs.community/2025-scs-team-container) - - +- [Meeting notes](https://input.scs.community/2025-scs-team-container) diff --git a/docs/overview.md b/docs/overview.md index 60d45ec4..b3b4448a 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,148 +1,149 @@ # Overview -## Cluster Stacks +## What is a Cluster Stack? -Cluster Stacks is a comprehensive framework and reference implementations for defining and managing Kubernetes clusters via the Cluster API. It is designed to cater to multiple providers and supports a broad range of Kubernetes versions, offering a standardized approach to managing and configuring Kubernetes clusters. +A cluster stack is a versioned bundle of everything needed to create and operate +a Kubernetes cluster via the Cluster API (CAPI). It packages three layers: -It encapsulates multiple layers, including node configuration, Cluster API setup, and application-level configurations, such as the Container Network Interface (CNI). By packaging these interdependent configurations, the cluster stack allows for efficient management and deployment of Kubernetes clusters, offering standardized, resilient, and self-managed Kubernetes environments. +1. **Cluster Class** -- a CAPI `ClusterClass` resource that defines the + infrastructure templates, machine configurations, and topology variables. +2. **Cluster Addons** -- Helm charts for core cluster components: CNI (Cilium), + Cloud Controller Manager, CSI driver, and metrics-server. +3. **Node Images** -- build instructions or references for the OS images that + run on cluster nodes. -## 🔧 Usage +These layers are published together as an OCI artifact. The +[Cluster Stack Operator](https://github.com/SovereignCloudStack/cluster-stack-operator) +(CSO) pulls these artifacts and installs them into a management cluster, making +the `ClusterClass` available for creating workload clusters. -Follow our [quickstart guide](providers/openstack/quickstart.md) for an introduction on how to deploy cluster stacks on openstack. +## Repository structure -## Layers of a Cluster Stack - -In essence, a cluster stack is an amalgamation of various components each of which serves a crucial role in setting up, maintaining, and operating a Kubernetes cluster. In the context of our framework, we categorize these components into three core layers: `cluster-class`, `cluster-addons`, and `node-images`. Let's delve deeper into understanding each of these layers: - -### 📚 Cluster Class - -The Cluster Class serves as a blueprint for creating and configuring Kubernetes clusters consistently. It encapsulates various aspects of a cluster, including: - -* The infrastructure provider details -* Networking configurations -* Cluster-class templating -* Other cluster-specific settings - -Essentially, it defines the desired configuration and properties of a Kubernetes cluster. It leverages the [ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) feature of Cluster API, which provides a declarative, Kubernetes-style API for cluster creation, configuration, and management. Any change in this layer or in the node-image or cluster-addon layers triggers a version bump in the cluster class, hence the cluster stack. - -### 🎁 Cluster Addons - -Cluster Addons are core components or services required for the Kubernetes cluster to function correctly and efficiently. These are not user-facing applications but rather foundational services critical to the operation and management of a Kubernetes cluster. They're usually installed and configured after the cluster infrastructure has been provisioned and before the cluster is ready to serve workloads. - -Cluster addons encompass a variety of functionalities, including but not limited to: - -* Container Network Interfaces (CNI): These are plugins that facilitate container networking. A CNI is integral to setting up network connectivity and ensuring communication between pods in a Kubernetes cluster. -* Cloud Controller Manager (CCM): The CCM is a Kubernetes control plane component that embeds the cloud-specific control logic. Its role is to manage the communication with the underlying cloud services. -* Konnectivity service: This is a network proxy that enables connectivity from the control plane to nodes and vice versa. It is a critical component that supports Kubernetes API server connectivity. -* Metrics Server: A cluster-wide aggregator of resource usage data, Metrics Server collects CPU, memory, and other metrics from nodes and pods, enabling features like Horizontal Pod Autoscaling. - -It's important to note that cluster addons are not user-provided applications or services that can be installed multiple times, such as ingress controllers, application-level monitoring tools, or user-facing APIs. Those are left to the discretion and responsibility of the users, who install and manage them according to their specific needs and preferences. - -Each addon version is independent and can be updated separately. However, a change in this layer also necessitates a version bump in the cluster class and the cluster stack, which is reflected in the metadata.yaml. - -### 🎞️ Node Images - -Node images provide the foundation for the operating system environment on each node of a Kubernetes cluster. They are typically a minimal operating system distribution, like a lightweight Linux distro, which may also include container runtime components such as Docker or containerd. - -Node images are responsible for providing the necessary environment and dependencies to support Kubernetes components and workloads. This includes components like kubelet, kube-proxy, and other necessary system utilities and libraries. - -The version of a node image can be different from that of the cluster stack or the cluster class. However, an update to a node image will trigger a version bump in the cluster class and hence the cluster stack. - -In the cluster-stacks repository's directory structure, the build instructions for Node Images are always placed within the respective directory. The instructions outline the steps and configurations required to create the Node Image automatically. The specific method for releasing the Node Image may vary based on the provider's capabilities and requirements. - -During the development phase, the build instructions serve as a reference within the repository itself. These instructions may utilize tools like Packer or other image-building techniques. This allows for flexibility and customization, enabling users to define their Node Images according to specific needs and requirements. - -However, when it comes to the release of the cluster stack, the Node Image can be provided in different ways depending on the capabilities of the provider or the desired deployment method. Here are a few examples: - -1. **URL on a remote endpoint**: In some cases, `providers` may support deploying a Node Image directly from a URL. In this scenario, the Node Image referenced in the `cluster stack`, specifically in the `cluster class`, would be provided as a URL pointing to a pre-built image accessible remotely. -1. **Artifact**: If the provider supports artifacts, the Node Image can be released as an artifact, such as a qcow2 file. The artifact would be uploaded to the provider, and the `cluster stack` references the artifact for node provisioning. -1. **Build Instructions**: In cases where the provider doesn't support direct URL deployment or artifact-based provisioning, the build instructions defined within the repository become critical. The build instructions serve as a comprehensive guide to build the Node Image, specifying all the necessary steps and configurations. - -Regardless of the release method, the cluster stack, specifically the cluster class, references the appropriate Node Image to be used for node provisioning. - -By allowing flexibility in the release and deployment methods of Node Images, the cluster stack framework caters to various provider capabilities and user requirements. This adaptability ensures the cluster stack can be deployed in diverse environments while maintaining a consistent and manageable approach to managing Kubernetes clusters. - -## 🌐 IaaS Provider, Kubernetes Service Provider, and Cluster API -In the context of the `cluster-stacks`, we distinguish between two types of providers: - -An **IaaS Provider**, in general, offers Infrastructure as a Service - providing the fundamental compute, storage, and network resources on which workloads can be run. In the context of cluster-stacks, an IaaS Provider specifically refers to an entity that owns an API for their infrastructure. If an organization uses a common infrastructure API, such as OpenStack, they are not considered an IaaS Provider in this context. However, if the organization owns the API for its infrastructure, it becomes an IaaS Provider for the purposes of cluster-stacks. +```text +providers/ + / + / + 1-XX/ # one directory per Kubernetes minor version + stack.yaml # metadata: provider, name, k8s version, addon pins + cluster-class/ # Helm chart producing the ClusterClass + cluster-addon/ # Helm chart with CNI, CCM, CSI, metrics-server + image-manager.yaml # OpenStack only: aggregated image references +``` -A **Kubernetes Service Provider**, on the other hand, is an entity that implements a cluster stack. They do so on top of the IaaS Providers, potentially spanning across multiple IaaS Providers. They use the IaaS Provider's infrastructure services and integrate them into their cluster stack implementations. +### Per-minor-version directories -The **Cluster API (CAPI)** is a Kubernetes project aimed at simplifying the process of managing Kubernetes clusters. It offers a declarative API that automates the creation, configuration, and management of clusters, providing a standardized way to interact with Kubernetes. The cluster stack approach leverages CAPI to deliver self-managed Kubernetes clusters. +Each `1-XX/` directory is completely self-contained. It carries its own +`stack.yaml`, ClusterClass templates, and addon charts. There is no inheritance +or sharing between minor versions -- changes to one version never affect another. -## 📌 Defining and Adding Providers -The structure of this repository is specifically designed to handle multiple providers, multiple cluster stacks per provider, and multiple Kubernetes versions per cluster stack. This organized structure allows us to effectively manage, develop, and maintain multiple cluster stacks across various Kubernetes versions and providers, all in a single repository. +This design makes it straightforward to: -### 📁 Repository Structure -The repository maintains a specific structure: +- Support provider-specific API and implementation differences across minors. +- Pin addons to version ranges that match the Kubernetes minor + (e.g. CCM `2.34.x` for K8s 1.34). +- Drop old versions by simply removing their directory. -* Each IaaS Provider has a directory under providers. -* Each IaaS Provider can have multiple cluster stack implementations. -* Each cluster stack supports multiple Kubernetes major and minor versions. +### stack.yaml -``` -providers/ -└── / - └── / - └── / -``` +Each version directory contains a `stack.yaml` that serves as the single source +of truth for that version: -The directory structure for adding a new provider would look something like this: +```yaml +provider: openstack +clusterStackName: scs +kubernetesVersion: 1.35 +addons: # version pins used by `just update addons` + ccm: 2.35.x + csi: 2.35.x ``` -providers/// -# example -providers/openstack/scs/1-28 -``` -This granular, hierarchical structure allows us to manage different versions of Kubernetes and their associated cluster stacks across different providers. -We decided to support multiple Kubernetes major and minor versions to provide the flexibility to accommodate different implementation requirements of the provider. However, we deliberately chose not to support Kubernetes patch versions directly. The reason is the high frequency of patch versions release (often weekly), which would complicate maintenance efforts significantly. +The `addons` section declares SemVer ranges. When you run `just update addons`, +the build system resolves these ranges against upstream Helm repositories and +updates the `Chart.yaml` dependencies in the addon chart. -Instead, we represent Kubernetes patch version updates through changes in our cluster stack version. For instance, if a patch version of Kubernetes necessitates a change in the node-image or the cluster-class configuration, this would trigger a version bump in the corresponding cluster stack, hence the cluster class, as reflected in the metadata.yaml. +## Available stacks -In this way, our versioning system, our directory structure, and our approach to Kubernetes versioning are all interlinked, providing us a comprehensive, manageable, and resilient framework for maintaining various Kubernetes distributions or cluster stacks across multiple providers and versions. +### OpenStack / scs -## 📑 Versioning +The standard SCS cluster stack. Creates dedicated VMs for both control plane and +worker nodes on OpenStack. Supports Kubernetes 1.32 through 1.35. -Note: This section is subject to change, as our new tool [csctl](https://github.com/SovereignCloudStack/csctl) will incorporate future versioning capabilities. +- Versions 1-32 through 1-35 use a `ClusterClass`-based configuration model. +- Version 1-35 uses unified variables (`flavor`) and + per-role overrides via `topology.controlPlane.variables.overrides`. -A fundamental aspect of the cluster stack approach is the encapsulation of versioning within a cluster stack distribution. Each of the components can be updated independently, leading to a flexible and maintainable system. +### OpenStack / hcp -However, the critical point to understand here is the relationship between these component versions and the cluster stack version. Whenever there's a change or an update to either the cluster addon or the node image, the version of the cluster stack must be bumped. And due to the connection between the cluster class and the cluster stack, the cluster class version must be updated to match the new cluster stack version. +Hosted Control Plane stack. The Kubernetes control plane runs as pods in the +management cluster (using the +[teutonet Hosted Control Plane provider](https://github.com/teutonet/cluster-api-provider-hosted-control-plane)); +only worker nodes are OpenStack VMs. -The cluster stack version doesn't simply mirror the versions of its components, but rather, it reflects the "version of change". In essence, the cluster stack version is a reflection of the state of the entire stack as a whole at a particular point in time. Any change in the components warrants a new state, and therefore a new version of the cluster stack. +See [providers/openstack/hcp.md](providers/openstack/hcp.md) for details. -So, an update to the cluster addon component will bump the version of the cluster stack, irrespective of the existing version of the node image. The same applies vice versa. When such an update occurs, the version of the cluster class is also incremented to align with the new cluster stack version, maintaining the unity of the cluster stack framework. +### Docker / scs -This versioning approach ensures a clear and precise track of changes, promoting efficient management, and isolated testing. It offers enhanced resilience for the Kubernetes distribution or the cluster stack, ensuring safe and secure upgrades even in rapid update cycles. It's an efficient method of maintaining stability in the rapidly changing environment of a Kubernetes stack. +A lightweight stack for local development and CI. Uses the CAPI Docker +infrastructure provider. No cloud credentials required. -The versioning of the cluster stack is primarily managed through a file named metadata.yaml, located at the root directory of each cluster stack. This file serves as the source of truth for the versioning information of the cluster stack, cluster class, node images, and cluster addons. +## Versioning -Here is an example of how metadata.yaml could look like: -``` -apiVersion: metadata.clusterstack.x-k8s.io/v1alpha1 -versions: - clusterStack: v3 - kubernetes: v1.27.3 - components: - clusterAddon: v2 - nodeImage: v1 -``` -In this example, the cluster stack (and thus the cluster class) is on version 3, while the cluster addon is on version 2 and node image is on version 1. +A cluster stack version is a single integer (`v1`, `v2`, ...) that represents +the state of the entire bundle at a point in time. Any change to the +ClusterClass, addons, or node images produces a new version. -When there's a change or update in the node images or cluster addons, we would bump the version of the cluster stack and cluster class, while leaving the unaffected component's version intact. So if the node image was updated, the metadata.yaml might then look like this: +The full identifier of a cluster stack release is: +```text +----v ``` -apiVersion: metadata.clusterstack.x-k8s.io/v1alpha1 -versions: - clusterStack: v4 - kubernetes: v1.27.3 - components: - clusterAddon: v2 - nodeImage: v2 -``` - -Here, the cluster stack and cluster class versions were updated to v4, the node image version was bumped to v2 due to the changes, while the cluster addon remained on v2 as it was not affected by the update. -This versioning approach allows us to keep track of changes across different components, manage these components effectively, and conduct isolated testing. This ensures that our Kubernetes distribution or cluster stack remains resilient, and we can perform safe and secure upgrades even in the face of rapid update cycles. The metadata.yaml plays a critical role in maintaining this structure and providing an accurate representation of the state of the whole stack at any given time. +For example: `openstack-scs-1-35-v3` means the third release of the `scs` stack +for Kubernetes 1.35 on OpenStack. + +Kubernetes patch versions are not part of the directory structure. A patch +version update (e.g. 1.35.1 to 1.35.2) is delivered by bumping the cluster +stack version, which updates the `kubernetesVersion` field and triggers a +rolling update of nodes. + +## Build system + +The build system uses [`just`](https://just.systems) as a task runner and a set +of bash scripts in `hack/`: + +| Command | Description | +| --------------------------------------- | ---------------------------------------------- | +| `just build --version 1.35` | Build locally to `.release/` | +| `just publish --version 1.35` | Build and push to OCI registry | +| `just dev --version 1.35` | Publish and print `ClusterStack` resource YAML | +| `just dev --install-cso --version 1.35` | Also install/upgrade CSO via Helm | +| `just install-cso` | Install CSO standalone | +| `just matrix` | Show version and addon matrix | +| `just update versions` | Update Kubernetes patch versions | +| `just update addons` | Update addon charts from upstream | +| `just generate-resources` | Generate `ClusterStack` + `Cluster` YAML | +| `just generate-docs` | Regenerate configuration documentation | +| `just clean` | Remove `.release/` build artifacts | + +### OCI workflow + +Cluster stack releases are OCI artifacts. For development, the build system +auto-configures [ttl.sh](https://ttl.sh) as a temporary registry (artifacts +expire after 24 hours). For production, set `OCI_REGISTRY` and +`OCI_REPOSITORY` environment variables to point to your registry. + +The CSO is installed via its Helm chart at +`oci://registry.scs.community/cluster-stacks/cso` and configured to pull from +whichever OCI registry you are publishing to. + +## Providers and the Cluster API + +The Cluster API (CAPI) provides a declarative, Kubernetes-native API for +creating and managing clusters. Each infrastructure provider (OpenStack, Docker, +etc.) implements the CAPI contracts for provisioning machines and networks. + +The CSO sits on top of CAPI: it manages the lifecycle of cluster stack releases, +installs ClusterClasses, and handles version upgrades. Users create `Cluster` +resources that reference a `ClusterClass` -- the CAPI topology controller then +reconciles the desired state into actual infrastructure. diff --git a/docs/providers/docker/scs-configuration.md b/docs/providers/docker/scs-configuration.md new file mode 100644 index 00000000..59be7fb9 --- /dev/null +++ b/docs/providers/docker/scs-configuration.md @@ -0,0 +1,56 @@ +# Configuration (Docker / scs) + +The Docker cluster stack is designed for local development and CI. It uses the +CAPI Docker infrastructure provider and requires no cloud credentials. + +## Version matrix + +| Version | K8s | CS Version | cilium | metrics-server | +|---------|-----|----------|---------|---------| +| 1-32 | 1.32 | - | 1.19.1 | 3.13.0 | +| 1-33 | 1.33 | - | 1.19.1 | 3.13.0 | +| 1-34 | 1.34 | - | 1.19.1 | 3.13.0 | +| 1-35 | 1.35 | - | 1.19.1 | 3.13.0 | + +## Example + +```yaml +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: my-docker-cluster + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + class: docker-scs-1-35-v1 + controlPlane: + replicas: 1 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 1 +``` + +## Available variables + +|Name|Type|Default|Example|Description|Required| +|----|----|-------|-------|-----------|--------| +|`imageRepository`|string|""|"registry.k8s.io"|Container registry to pull images from. If empty, the kubeadm default is used.|False| +|`certSANs`|array|[]|["mydomain.example"]|Extra Subject Alternative Names for the API Server signing certificate.|False| +|`oidcConfig.clientID`|string||"kubectl"|OIDC client ID for API server authentication.|| +|`oidcConfig.issuerURL`|string||"https://dex.example.com"|OIDC provider discovery URL (must be HTTPS).|| +|`oidcConfig.usernameClaim`|string|"preferred_username"|"email"|JWT claim to use as the username.|| +|`oidcConfig.groupsClaim`|string|"groups"|"groups"|JWT claim to use as groups.|| +|`oidcConfig.usernamePrefix`|string|"oidc:"|"oidc:"|Prefix for OIDC usernames.|| +|`oidcConfig.groupsPrefix`|string|"oidc:"|"oidc:"|Prefix for OIDC group names.|| +|`registryMirrors`|array|[]|[{"hostnameUpstream": "docker.io", "urlMirror": "https://mirror.example.com"}]|Container registry mirrors for node containerd/CRI-O configuration.|| diff --git a/docs/providers/openstack/configuration.md b/docs/providers/openstack/configuration.md deleted file mode 100644 index f72b29f2..00000000 --- a/docs/providers/openstack/configuration.md +++ /dev/null @@ -1,84 +0,0 @@ -# Configuration - -This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. - -## Example - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: - namespace: - labels: - managed-secret: cloud-config -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - serviceDomain: cluster.local - services: - cidrBlocks: - - 10.96.0.0/12 - topology: - variables: // <-- variables from the table can be set here - - name: controller_flavor - value: "SCS-4V-8-20" - - name: worker_flavor - value: "SCS-4V-8-20" - - name: external_id - value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" - class: openstack-alpha-1-29-v2 - controlPlane: - replicas: 2 - version: v1.29.3 - workers: - machineDeployments: - - class: openstack-alpha-1-29-v2 - failureDomain: nova - name: openstack-alpha-1-29-v2 - replicas: 4 -``` - -Variables from the table containing a `.` are to be used in an object with the part before the dot being the object name and the part behind the dot being the value names. The following example demonstrates this with `oidc_config`. - -```yaml ---- -topology: - variables: - - name: oidc_config - value: - issuer_url: "https://dex.k8s.scs.community" - client_id: "kubectl" -``` - -## Available variables - - -| Name | Type | Default | Example | Description | Required | -| ---------------------------------- | ------- | -------------------------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `external_id` | string | "" | "ebfe5546-f09f-4f42-ab54-094e457d42ec" | ExternalNetworkID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs. | False | -| `controller_flavor` | string | "SCS-2V-4-20s" | "SCS-2V-4-20s" | OpenStack instance flavor for control-plane nodes. | False | -| `worker_flavor` | string | "SCS-2V-4" | "SCS-2V-4" | OpenStack instance flavor for worker nodes. | False | -| `controller_root_disk` | integer | | 25 | Root disk size in GiB for control-plane nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should only be used for the diskless flavors. | False | -| `worker_root_disk` | integer | 25 | 25 | Root disk size in GiB for worker nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should be used for the diskless flavors. | False | -| `openstack_security_groups` | array | [] | ['security-group-1'] | The names of the security groups to assign to the instance | False | -| `cloud_name` | string | "openstack" | "openstack" | The name of the cloud to use from the clouds secret | False | -| `secret_name` | string | "openstack" | "openstack" | The name of the clouds secret | False | -| `controller_server_group_id` | string | "" | "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" | The server group to assign the control plane nodes to. | False | -| `worker_server_group_id` | string | "" | "869fe071-1e56-46a9-9166-47c9f228e297" | The server group to assign the worker nodes to. | False | -| `ssh_key` | string | "" | "capi-keypair" | The ssh key to inject in the nodes. | False | -| `apiserver_loadbalancer` | string | "octavia-amphora" | "none, octavia-amphora, octavia-ovn" | "In this cluster-stack we have two kind of loadbalancers. Each of them has its own configuration variable. This setting here is to configure the loadbalancer that is placed in front of the apiserver.
You can choose from 2 options:

none:
No loadbalancer solution will be deployed

octavia-amphora:
(default) Uses openstack's loadbalancer service (provider:amphora)

octavia-ovn:
Uses openstack's loadbalancer service (provider:ovn)
| False | -| `dns_nameservers` | array | ['5.1.66.255', '185.150.99.255'] | ['5.1.66.255', '185.150.99.255'] | "DNSNameservers is the list of nameservers for the OpenStack Subnet being created. Set this value when you need to create a new network/subnet while the access through DNS is required.
| False | -| `node_cidr` | string | "10.8.0.0/20" | "10.8.0.0/20" | "NodeCIDR is the OpenStack Subnet to be created. Cluster actuator will create a network, a subnet with NodeCIDR, and a router connected to this subnet. If you leave this empty, no network will be created.
| False | -| `certSANs` | array | [] | ['mydomain.example'] | CertSANs sets extra Subject Alternative Names for the API Server signing cert. | False | -| `oidc_config.client_id` | string | | kubectl | A client id that all tokens must be issued for. | | -| `oidc_config.issuer_url` | string | | `https://dex.example.com` | URL of the provider that allows the API server to dis cover public signing keys. Only URLs that use the https:// scheme are acc epted. This is typically the provider's discovery URL, changed to have an emp ty path | | -| `oidc_config.username_claim` | string | preferred_username | preferred_username | JWT claim to use as the user name. By default sub, whi ch is expected to be a unique identifier of the end user. Admins can choose oth er claims, such as email or name, depending on their provider. However, cla ims other than email will be prefixed with the issuer URL to prevent naming cla shes with other plugins. | | -| `oidc_config.groups_claim` | string | groups | groups | JWT claim to use as the user's group. If the claim is present it must be an array of strings. | | -| `oidc_config.username_prefix` | string | oidc: | oidc: | Prefix prepended to username claims to prevent cla shes with existing names (such as system: users). For example, the value oid c: will create usernames like oidc:jane.doe. If this flag isn't provided and --o idc-username-claim is a value other than email the prefix defaults to ( Iss uer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value - c an be used to disable all prefixing. | | -| `oidc_config.groups_prefix` | string | oidc: | oidc: | Prefix prepended to group claims to prevent clashes wit h existing names (such as system: groups). For example, the value oidc: will cre ate group names like oidc:engineering and oidc:infra. | | -| `network_mtu` | integer | | 1500 | NetworkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID. | False | -| `controlPlaneAvailabilityZones` | array | | ['nova'] | ControlPlaneAvailabilityZones is the set of availability zones which control plane machines may be deployed to. | False | -| `controlPlaneOmitAvailabilityZone` | boolean | | True | ControlPlaneOmitAvailabilityZone causes availability zone to be omitted when creating control plane nodes, allowing the Nova scheduler to make a decision on which availability zone to use based on other scheduling constraints. | False | diff --git a/docs/providers/openstack/hcp.md b/docs/providers/openstack/hcp.md new file mode 100644 index 00000000..3b433f3c --- /dev/null +++ b/docs/providers/openstack/hcp.md @@ -0,0 +1,153 @@ +# OpenStack HCP (Hosted Control Plane) + +The `hcp` cluster stack runs the Kubernetes control plane as pods inside the +management cluster, using the +[teutonet Hosted Control Plane provider](https://github.com/teutonet/cluster-api-provider-hosted-control-plane). +Only worker nodes are created as OpenStack VMs. + +## How it differs from the `scs` stack + +| Aspect | scs | hcp | +| ------ | --- | --- | +| Control plane | Dedicated OpenStack VMs | Pods in the management cluster | +| Control plane template | `KubeadmControlPlaneTemplate` | `HostedControlPlaneTemplate` (v1alpha1) | +| CAPI API version | `v1beta2` (1-35) | `v1beta1` | +| CP machine infrastructure | `OpenStackMachineTemplate` | None (no CP VMs) | +| API server load balancer | Configurable (Octavia / none) | `none` by default (uses Gateway API) | +| API server floating IP | Configurable | Disabled by default | +| Variable model | Unified (`flavor`, `rootDisk`) | Worker-prefixed (`workerFlavor`, etc.) | +| Unique variables | `oidcConfig`, `registryMirrors` | `gatewayName`, `gatewayNamespace`, `controlPlaneReplicas` | + +## Architecture + +```text +Management Cluster + +-- CP pods (etcd, apiserver, controller-manager, scheduler) + +-- HostedControlPlane resource + +-- Gateway API ingress (optional) + +OpenStack + +-- Worker VMs only + +-- Cilium CNI + +-- CCM + CSI (as DaemonSets/Deployments on workers) +``` + +The control plane is exposed to worker nodes via the management cluster's +network. For external access to the API server, configure Gateway API resources +using the `gatewayName` and `gatewayNamespace` variables. + +## Prerequisites + +In addition to the standard [quickstart prerequisites](../../quickstart.md): + +1. Install the teutonet Hosted Control Plane provider in the management cluster: + + ```bash + kubectl apply -f https://github.com/teutonet/cluster-api-provider-hosted-control-plane/releases/latest/download/install.yaml + ``` + +2. (Optional) Set up a Gateway API implementation (e.g., Envoy Gateway, Cilium + Gateway API) if you want external access to the hosted API servers. + +## Available versions + +| Version | Kubernetes | Notes | +|---------|------------|----------------------------| +| 1-33 | 1.33 | Initial HCP release | +| 1-34 | 1.34 | Updated CCM/CSI | +| 1-35 | 1.35 | Updated CCM/CSI | + +## Usage + +### Build and publish + +```bash +export PROVIDER=openstack +export CLUSTER_STACK=hcp + +just dev --install-cso --version 1.35 +``` + +### Create a cluster + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: my-hcp-cluster + namespace: my-tenant + labels: + managed-secret: cloud-config +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + class: openstack-hcp-1-35-v1 + controlPlane: + replicas: 3 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 3 + variables: + - name: workerFlavor + value: "SCS-4V-8-20" + - name: networkExternalID + value: "your-external-network-uuid" +``` + +### With Gateway API ingress + +To expose the hosted API server externally, configure a Gateway resource in the +management cluster and reference it: + +```yaml + variables: + - name: gatewayName + value: "cluster-gateway" + - name: gatewayNamespace + value: "gateway-system" +``` + +## Configuration variables + +### HCP-specific variables + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `controlPlaneReplicas` | integer | 3 | Number of hosted control plane replicas | +| `gatewayName` | string | "" | Gateway API resource name for API server ingress | +| `gatewayNamespace` | string | "default" | Namespace of the Gateway resource | +| `disableAPIServerFloatingIP` | boolean | true | Disable floating IP for API server | +| `apiServerLoadBalancer` | string | "none" | Load balancer type (typically `none` for HCP) | + +### Worker variables + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `workerFlavor` | string | "SCS-2V-4" | OpenStack flavor for worker nodes | +| `workerRootDisk` | integer | 25 | Root disk size in GiB | +| `workerServerGroupID` | string | "" | Anti-affinity server group for workers | +| `workerAdditionalBlockDevices` | array | [] | Extra Cinder volumes for workers | +| `workerSecurityGroups` | array | [] | Security group names for workers | +| `workerSecurityGroupIDs` | array | [] | Security group UUIDs for workers | + +### Shared variables (same as scs) + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `imageName` | string | "ubuntu-capi-image" | Base OS image name | +| `networkExternalID` | string | "" | External network UUID | +| `networkMTU` | integer | (provider default) | MTU for cluster network | +| `dnsNameservers` | array | ["9.9.9.9", "149.112.112.112"] | DNS nameservers | +| `nodeCIDR` | string | "10.8.0.0/20" | Subnet CIDR for cluster nodes | +| `sshKeyName` | string | "" | SSH key to inject into nodes | +| `certSANs` | array | [] | Extra SANs for API server certificate | diff --git a/docs/providers/openstack/quickstart.md b/docs/providers/openstack/quickstart.md deleted file mode 100644 index 00bccef8..00000000 --- a/docs/providers/openstack/quickstart.md +++ /dev/null @@ -1,290 +0,0 @@ -# Quickstart - -This quickstart guide contains steps to install the [Cluster Stack Operator][CSO] (CSO) utilizing the [Cluster Stack Provider OpenStack][CSPO] (CSPO) to provide [ClusterClasses][ClusterClass] which can be used with the [Kubernetes Cluster API][CAPI] to create Kubernetes Clusters. - -This section guides you through all the necessary steps to create a workload Kubernetes cluster on top of the OpenStack infrastructure. The guide describes a path that utilizes the `clusterctl` CLI tool to manage the lifecycle of a CAPI management cluster and employs `kind` to create a local non-production managemnt cluster. - -Note that it is a common practice to create a temporary, local [bootstrap cluster](https://cluster-api.sigs.k8s.io/reference/glossary#bootstrap-cluster) which is then used to provision a target [management cluster](https://cluster-api.sigs.k8s.io/reference/glossary#management-cluster) on the selected infrastructure. - -## Prerequisites - -- Install [Docker](https://docs.docker.com/get-docker/) and [kind](https://helm.sh/docs/intro/install/) -- Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- Install [Helm](https://helm.sh/docs/intro/install/) -- Install [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl) -- Install [go](https://go.dev/doc/install) -- Install [jq](https://jqlang.github.io/jq/) - -## Initialize the management cluster - -Create the kind cluster: - -```bash -kind create cluster -``` - -Transform the Kubernetes cluster into a management cluster by using `clusterctl init` and bootstrap it with CAPI and Cluster API Provider OpenStack ([CAPO](https://github.com/kubernetes-sigs/cluster-api-provider-openstack)) components: - -```bash -export CLUSTER_TOPOLOGY=true -export EXP_CLUSTER_RESOURCE_SET=true -export EXP_RUNTIME_SDK=true -clusterctl init --infrastructure openstack -kubectl apply -f https://github.com/k-orc/openstack-resource-controller/releases/latest/download/install.yaml -``` - -Note that the manual deployment of the openstack resource controller (ORC) is required since capo-0.12. If you use `clusterctl upgrade` to upgrade capo from earlier version, you'll also need to manually add ORC to the management host/cluster. - -### CSO and CSPO variables preparation (CSP) - -The CSO and CSPO must be directed to the Cluster Stacks repository housing releases for the OpenStack provider. -Modify and export the following environment variables if you wish to redirect CSO and CSPO to an alternative Git repository - -Be aware that GitHub enforces limitations on the number of API requests per unit of time. To overcome this, -it is recommended to configure a [personal access token](https://github.com/settings/personal-access-tokens/new) for authenticated calls. This will significantly increase the rate limit for GitHub API requests. -Fine grained PAT with `Public Repositories (read-only)` is enough. - -```bash -export GIT_PROVIDER_B64=Z2l0aHVi # github -export GIT_ORG_NAME_B64=U292ZXJlaWduQ2xvdWRTdGFjaw== # SovereignCloudStack -export GIT_REPOSITORY_NAME_B64=Y2x1c3Rlci1zdGFja3M= # cluster-stacks -export GIT_ACCESS_TOKEN_B64=$(echo -n '' | base64 -w0) -``` - -### CSO and CSPO deployment (CSP) - -Install the [envsubst](https://github.com/drone/envsubst) Go package. It is required to enable the expansion of variables specified in CSPO and CSO manifests. - -```bash -GOBIN=/tmp go install github.com/drone/envsubst/v2/cmd/envsubst@latest -``` -Note: On typical Linux distros, you will have a binary `/usr/bin/envsubst` from the gettext package that does *not* work. - -Get the latest CSO release version and apply CSO manifests to the management cluster. - -```bash -# Get the latest CSO release version and apply CSO manifests -curl -sSL https://github.com/SovereignCloudStack/cluster-stack-operator/releases/latest/download/cso-infrastructure-components.yaml | /tmp/envsubst | kubectl apply -f - -``` - -Get the latest CSPO release version and apply CSPO manifests to the management cluster. - -```bash -# Get the latest CSPO release version and apply CSPO manifests -curl -sSL https://github.com/sovereignCloudStack/cluster-stack-provider-openstack/releases/latest/download/cspo-infrastructure-components.yaml | /tmp/envsubst | kubectl apply -f - -``` - -## Define a namespace for a tenant (CSP/per tenant) - -```sh -export CS_NAMESPACE=my-tenant -``` - -### Deploy CSP-helper chart - -The csp-helper chart is meant to create per tenant credentials as well as the tenants namespace where all resources for this tenant will live in. - -Cloud and secret name default to `openstack`. - -Example `clouds.yaml` - -```yaml -clouds: - openstack: - auth: - auth_url: https://api.gx-scs.sovereignit.cloud:5000/v3 - application_credential_id: "" - application_credential_secret: "" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - auth_type: "v3applicationcredential" -``` - -```bash -helm upgrade -i csp-helper-"${CS_NAMESPACE}" -n "${CS_NAMESPACE}" --create-namespace https://github.com/SovereignCloudStack/openstack-csp-helper/releases/latest/download/openstack-csp-helper.tgz -f path/to/clouds.yaml -``` - -## Create Cluster Stack definition (CSP/per tenant) - -Configure the Cluster Stack you want to use: - -```sh -# the name of the cluster stack (must match a name of a directory in https://github.com/SovereignCloudStack/cluster-stacks/tree/main/providers/openstack) -export CS_NAME=scs - -# the kubernetes version of the cluster stack (must match a tag for the kubernetes version and the stack version) -export CS_K8S_VERSION=1.29 - -# the version of the cluster stack (must match a tag for the kubernetes version and the stack version) -export CS_VERSION=v1 -export CS_CHANNEL=stable - -# must match a cloud section name in the used clouds.yaml -export CS_CLOUDNAME=openstack -export CS_SECRETNAME="${CS_CLOUDNAME}" -``` - -This will use the cluster-stack as defined in the `providers/openstack/scs` directory. - -```bash -cat >clusterstack.yaml < cluster.yaml < kubeconfig.yaml -# Communicate with the workload cluster -kubectl --kubeconfig kubeconfig.yaml get nodes -``` - -## Check the workload cluster health - -```bash -$ kubectl --kubeconfig kubeconfig.yaml get pods -A -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system cilium-8mzrx 1/1 Running 0 7m58s -kube-system cilium-jdxqm 1/1 Running 0 6m43s -kube-system cilium-operator-6bb4c7d6b6-c77tn 1/1 Running 0 7m57s -kube-system cilium-operator-6bb4c7d6b6-l2df8 1/1 Running 0 7m58s -kube-system cilium-p9tkv 1/1 Running 0 6m44s -kube-system cilium-thbc8 1/1 Running 0 6m45s -kube-system coredns-5dd5756b68-k68j4 1/1 Running 0 8m3s -kube-system coredns-5dd5756b68-vjg9r 1/1 Running 0 8m3s -kube-system etcd-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-apiserver-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-controller-manager-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-proxy-54f8w 1/1 Running 0 6m44s -kube-system kube-proxy-8z8kb 1/1 Running 0 6m43s -kube-system kube-proxy-jht46 1/1 Running 0 8m3s -kube-system kube-proxy-mt69p 1/1 Running 0 6m45s -kube-system kube-scheduler-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system metrics-server-6578bd6756-vztzf 1/1 Running 0 7m57s -kube-system openstack-cinder-csi-controllerplugin-776696786b-ksf77 6/6 Running 0 7m57s -kube-system openstack-cinder-csi-nodeplugin-96dlg 3/3 Running 0 6m43s -kube-system openstack-cinder-csi-nodeplugin-crhc4 3/3 Running 0 6m44s -kube-system openstack-cinder-csi-nodeplugin-d7rzz 3/3 Running 0 7m58s -kube-system openstack-cinder-csi-nodeplugin-nkgq6 3/3 Running 0 6m44s -kube-system openstack-cloud-controller-manager-hp2n2 1/1 Running 0 7m9s -``` - -[CAPI]: https://cluster-api.sigs.k8s.io/ -[CSO]: https://github.com/sovereignCloudStack/cluster-stack-operator/ -[CSPO]: https://github.com/SovereignCloudStack/cluster-stacks/tree/main/providers/openstack -[ClusterClass]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210526-cluster-class-and-managed-topologies.md diff --git a/docs/providers/openstack/scs-configuration.md b/docs/providers/openstack/scs-configuration.md new file mode 100644 index 00000000..bdbeaea1 --- /dev/null +++ b/docs/providers/openstack/scs-configuration.md @@ -0,0 +1,105 @@ +# Configuration + +This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. + +## Version matrix + +| Version | K8s | CS Version | cilium | metrics-server | os-csi | os-ccm | +| ------- | --- | ---------- | ------ | -------------- | ------ | ------ | +| 1-32 | 1.32 | - | 1.19.1 | 3.13.0 | 2.32.x | 2.32.x | +| 1-33 | 1.33 | - | 1.19.1 | 3.13.0 | 2.33.x | 2.33.x | +| 1-34 | 1.34 | - | 1.19.1 | 3.13.0 | 2.34.x | 2.34.x | +| 1-35 | 1.35 | - | 1.19.1 | 3.13.0 | 2.35.x | 2.35.x | + +## Example + +```yaml +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: my-cluster + namespace: my-namespace + labels: + managed-secret: cloud-config +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + variables: # <-- variables from the table below can be set here + - name: flavor + value: "SCS-4V-8-20" + - name: networkExternalID + value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + class: openstack-scs-1-35-v1 + controlPlane: + replicas: 3 + variables: + overrides: + - name: flavor + value: "SCS-4V-8-50" + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 3 +``` + +Variables of type `object` are set as nested values. The following example demonstrates this with `oidcConfig`: + +```yaml +... +topology: + variables: + - name: oidcConfig + value: + issuerURL: "https://dex.k8s.scs.community" + clientID: "kubectl" +... +``` + +In v1beta2, per-role overrides (e.g. different flavors for control plane and workers) are set via `topology.controlPlane.variables.overrides` and `topology.workers.machineDeployments[].variables.overrides` instead of separate variable names. + +## Available variables + +> **Note:** This table documents the **1-35** (v1beta2) variable set with unified +> variable names. The other currently maintained SCS minors in this repo use the +> same unified variable model and `ClusterClass`-based configuration pattern. + +| Name | Type | Default | Example | Description | Required | +| ---- | ---- | ------- | ------- | ----------- | -------- | +| `imageName` | string | "ubuntu-capi-image" | "ubuntu-capi-image" | Base name of the OpenStack image for cluster nodes. | False | +| `imageIsOrc` | boolean | false | true | Whether the image name refers to an ORC (OpenStack Resource Controller) image resource instead of a Glance image. | False | +| `imageAddVersion` | boolean | true | false | Append the Kubernetes version suffix to the image name (e.g. `ubuntu-capi-image-v1.35`). | False | +| `networkExternalID` | string | "" | "ebfe5546-f09f-4f42-ab54-094e457d42ec" | ID of the external OpenStack network for public internet access. | False | +| `networkMTU` | integer | (provider default) | 1500 | Maximum transmission unit (MTU) for the private cluster network. | False | +| `dnsNameservers` | array | ["9.9.9.9", "149.112.112.112"] | ["8.8.8.8"] | DNS nameservers for the cluster subnet. | False | +| `nodeCIDR` | string | "10.8.0.0/20" | "10.8.0.0/20" | CIDR for the cluster subnet. A network, subnet, and router will be created. | False | +| `flavor` | string | "SCS-2V-4" | "SCS-4V-8-20" | OpenStack instance flavor for all nodes. Override per role using topology variable overrides. | False | +| `rootDisk` | integer | 50 | 50 | Root disk size in GiB. When set, an OpenStack volume is used instead of the ephemeral disk from the flavor. | False | +| `serverGroupID` | string | "" | "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" | Server group for anti-affinity placement. Override per role using topology variable overrides. | False | +| `additionalBlockDevices` | array | [] | [{"name": "data", "sizeGiB": 100, "type": "Volume"}] | Additional Cinder volumes to attach to nodes. | False | +| `sshKeyName` | string | "" | "capi-keypair" | SSH key pair name to inject into nodes. | False | +| `securityGroups` | array | [] | ["security-group-1"] | Extra security groups by name for all nodes. Ignored if `securityGroupIDs` is set. | False | +| `securityGroupIDs` | array | [] | ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] | Extra security groups by UUID for all nodes. Takes precedence over `securityGroups`. | False | +| `apiServerLoadBalancer` | string | "octavia-ovn" | "none" | Load balancer for the API server. Options: `none`, `octavia-amphora`, `octavia-ovn`. | False | +| `apiServerAllowedCIDRs` | array | [] | ["10.0.0.0/8"] | CIDRs allowed to access the API server load balancer. | False | +| `disableAPIServerFloatingIP` | boolean | false | true | Disable floating IP for the API server. | False | +| `certSANs` | array | [] | ["mydomain.example"] | Extra Subject Alternative Names for the API server certificate. | False | +| `controlPlaneAvailabilityZones` | array | [] | ["nova"] | Availability zones for control plane nodes. | False | +| `controlPlaneOmitAvailabilityZone` | boolean | false | true | Omit availability zone when creating control plane nodes, letting Nova schedule freely. | False | +| `identityRef.name` | string | "openstack" | "openstack" | Name of the Secret containing OpenStack credentials. | False | +| `identityRef.cloudName` | string | "openstack" | "openstack" | Cloud name within the credentials Secret. | False | +| `oidcConfig.clientID` | string | "" | "kubectl" | OIDC client ID for API server authentication. | False | +| `oidcConfig.issuerURL` | string | "" | `"https://dex.example.com"` | OIDC provider discovery URL (must be HTTPS). | False | +| `oidcConfig.usernameClaim` | string | "preferred_username" | "email" | JWT claim to use as the username. | False | +| `oidcConfig.groupsClaim` | string | "groups" | "groups" | JWT claim to use as groups. | False | +| `oidcConfig.usernamePrefix` | string | "oidc:" | "oidc:" | Prefix for OIDC usernames. | False | +| `oidcConfig.groupsPrefix` | string | "oidc:" | "oidc:" | Prefix for OIDC group names. | False | +| `registryMirrors` | array | [] | [{"hostnameUpstream": "docker.io", "urlMirror": "https://mirror.example.com"}] | Container registry mirrors for node containerd configuration. | False | diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..59015424 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,270 @@ +# Quickstart + +This guide walks through creating a CAPI management cluster and deploying a +workload cluster using one of the available cluster stacks. It covers OpenStack, +Docker (local development), and HCP (Hosted Control Plane) variants. + +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) and + [kind](https://kind.sigs.k8s.io/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [Helm](https://helm.sh/docs/intro/install/) +- [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl) +- [just](https://just.systems/man/en/installation.html) + +For OpenStack stacks you also need: + +- An OpenStack cloud with application credentials +- A `clouds.yaml` with your credentials + +## 1. Create a management cluster + +```bash +kind create cluster +``` + +## 2. Install CAPI and the infrastructure provider + +### OpenStack (scs and hcp stacks) + +```bash +clusterctl init --infrastructure openstack +``` + +For clusters using the [OpenStack Resource Controller](https://github.com/k-orc/openstack-resource-controller) +image format (ORC), also install ORC: + +```bash +kubectl apply -f https://github.com/k-orc/openstack-resource-controller/releases/latest/download/install.yaml +``` + +### Docker (local development) + +```bash +clusterctl init --infrastructure docker +``` + +### HCP (Hosted Control Plane) + +The HCP stack requires both the OpenStack provider and the teutonet Hosted +Control Plane provider: + +```bash +clusterctl init --infrastructure openstack +# Install the Hosted Control Plane provider (see teutonet docs for latest URL) +kubectl apply -f https://github.com/teutonet/cluster-api-provider-hosted-control-plane/releases/latest/download/install.yaml +``` + +## 3. Install the Cluster Stack Operator + +The quickest way during development: + +```bash +just install-cso +``` + +This installs the CSO Helm chart from +`oci://registry.scs.community/cluster-stacks/cso` and auto-configures +[ttl.sh](https://ttl.sh) as a temporary OCI registry. + +For production, set `OCI_REGISTRY` and `OCI_REPOSITORY` before running: + +```bash +export OCI_REGISTRY=registry.example.com +export OCI_REPOSITORY=kaas/cluster-stacks +just install-cso +``` + +## 4. Build and publish a cluster stack + +```bash +# Build and publish all versions for the default stack (openstack/scs) +just publish --all + +# Or target a specific version +just publish --version 1.35 + +# For a different stack, set the environment +PROVIDER=docker CLUSTER_STACK=scs just publish --version 1.35 +``` + +The `dev` recipe combines publish with generating the `ClusterStack` resource: + +```bash +just dev --version 1.35 +``` + +This prints a `ClusterStack` YAML that you can pipe to `kubectl apply -f -`. + +To also install/upgrade the CSO in one step: + +```bash +just dev --install-cso --version 1.35 +``` + +## 5. Deploy OpenStack credentials (OpenStack only) + +Create a namespace for your tenant and deploy credentials using the +[csp-helper chart](https://github.com/SovereignCloudStack/openstack-csp-helper): + +```bash +export CS_NAMESPACE=my-tenant + +helm upgrade -i csp-helper-"${CS_NAMESPACE}" \ + -n "${CS_NAMESPACE}" --create-namespace \ + https://github.com/SovereignCloudStack/openstack-csp-helper/releases/latest/download/openstack-csp-helper.tgz \ + -f path/to/clouds.yaml +``` + +## 6. Apply the ClusterStack resource + +If you used `just dev`, the output already contains this. Otherwise, generate it: + +```bash +just generate-resources --version 1.35 --clusterstack-only +``` + +Example output: + +```yaml +apiVersion: clusterstack.x-k8s.io/v1alpha1 +kind: ClusterStack +metadata: + name: openstack-1-35 + namespace: cluster +spec: + provider: openstack + name: scs + kubernetesVersion: "1.35" + channel: custom + autoSubscribe: false + versions: + - v1 # matches your published version +``` + +Apply it and wait for the `ClusterClass` to become available: + +```bash +kubectl apply -f clusterstack.yaml +kubectl get clusterclass -w +``` + +## 7. Create a workload cluster + +Generate a `Cluster` resource: + +```bash +just generate-resources --version 1.35 --cluster-only +``` + +Or write one manually. Here is a minimal example for OpenStack/scs 1-35: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: my-cluster + namespace: my-tenant + labels: + managed-secret: cloud-config +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + class: openstack-scs-1-35-v1 + controlPlane: + replicas: 3 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 3 + variables: + - name: flavor + value: "SCS-4V-8-20" + - name: networkExternalID + value: "your-external-network-uuid" +``` + +Apply and monitor: + +```bash +kubectl apply -f cluster.yaml +clusterctl -n my-tenant describe cluster my-cluster +``` + +## 8. Access the workload cluster + +```bash +clusterctl -n my-tenant get kubeconfig my-cluster > kubeconfig.yaml +kubectl --kubeconfig kubeconfig.yaml get nodes +``` + +## Docker variant + +For the Docker provider, no cloud credentials or external network are needed: + +```bash +export PROVIDER=docker +export CLUSTER_STACK=scs + +just dev --install-cso --version 1.35 +# Apply the printed ClusterStack YAML, then create a cluster +just generate-resources --version 1.35 --cluster-only | kubectl apply -f - +``` + +## HCP variant + +The HCP stack creates control plane pods in the management cluster instead of +dedicated VMs. Worker-specific variables use the `worker` prefix +(`workerFlavor`, `workerRootDisk`): + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: my-hcp-cluster + namespace: my-tenant + labels: + managed-secret: cloud-config +spec: + topology: + class: openstack-hcp-1-35-v1 + controlPlane: + replicas: 3 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 3 + variables: + - name: workerFlavor + value: "SCS-4V-8-20" + - name: networkExternalID + value: "your-external-network-uuid" + - name: gatewayName + value: "my-gateway" + - name: gatewayNamespace + value: "default" +``` + +See [providers/openstack/hcp.md](providers/openstack/hcp.md) for full details. + +## Useful commands + +```bash +just matrix # show version/addon matrix for all stacks +just update versions # update Kubernetes patch versions +just update addons # update addon chart versions +just generate-docs # regenerate configuration reference docs +just clean # remove build artifacts +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..6714924d --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1768886240, + "narHash": "sha256-C2TjvwYZ2VDxYWeqvvJ5XPPp6U7H66zeJlRaErJKoEM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "80e4adbcf8992d3fd27ad4964fbb84907f9478b0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..8db9a34d --- /dev/null +++ b/flake.nix @@ -0,0 +1,88 @@ +{ + description = "Cluster Stacks - Build tools for SCS Kubernetes cluster stacks"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + devShells.default = pkgs.mkShell { + name = "cluster-stacks-dev"; + + buildInputs = with pkgs; [ + # Core tools + bash + git + curl + + # Container tools + docker + podman + + # Kubernetes tools + kubectl + kubernetes-helm + kind + kustomize + + # Build tools + just + python3 + python3Packages.pyyaml + jq + yq-go + + # OCI/Registry tools + oras + ]; + + shellHook = '' + echo "Cluster Stacks development environment" + echo "" + echo "Available tools:" + echo " just - Run 'just --list' to see available commands" + echo " helm - Kubernetes package manager" + echo " kubectl - Kubernetes CLI" + echo " kind - Local Kubernetes clusters" + echo " yq - YAML processor" + echo " oras - OCI registry client" + echo " python3 - With PyYAML" + echo "" + + # Generate shell completions into a cache directory + comp_dir="''${XDG_CACHE_HOME:-$HOME/.cache}/cluster-stacks/completions" + mkdir -p "$comp_dir" + + user_shell=$(getent passwd "$(whoami)" 2>/dev/null | cut -d: -f7) + shell_name=$(basename "''${user_shell:-bash}") + + # Regenerate completions if the directory is empty or tools were updated + if [ ! -f "$comp_dir/.$shell_name-generated" ]; then + kubectl completion "$shell_name" > "$comp_dir/_kubectl" 2>/dev/null || true + helm completion "$shell_name" > "$comp_dir/_helm" 2>/dev/null || true + just --completions "$shell_name" > "$comp_dir/_just" 2>/dev/null || true + kind completion "$shell_name" > "$comp_dir/_kind" 2>/dev/null || true + oras completion "$shell_name" > "$comp_dir/_oras" 2>/dev/null || true + touch "$comp_dir/.$shell_name-generated" + fi + + # Make completions available + export FPATH="$comp_dir:$FPATH" + + # Start user's preferred shell instead of bash. + # nix develop always drops into bash; this detects the user's + # real login shell from /etc/passwd and exec's into it. + if [ -n "$user_shell" ] && [ "$shell_name" != "bash" ] && [ -x "$user_shell" ]; then + exec "$user_shell" + fi + ''; + }; + } + ); +} diff --git a/hack/build.sh b/hack/build.sh new file mode 100755 index 00000000..d09cc23c --- /dev/null +++ b/hack/build.sh @@ -0,0 +1,621 @@ +#!/usr/bin/env bash +# Build and optionally publish cluster-stack release artifacts. +# +# Usage: +# ./hack/build.sh [stack-dir] [options] +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). +# +# Options: +# --version Build for a specific K8s minor version (e.g., 1.34) +# --all Build for all K8s versions (all 1-* subdirs) +# --publish Push to OCI registry after building +# --install-cso Install/upgrade the Cluster Stack Operator with matching OCI config +# --validate Validate addon bundle structure against clusteraddon.yaml +# +# Without --version or --all, builds the highest 1-* subdirectory. +# +# Each 1-XX/ subdir must contain a stack.yaml with at minimum: +# provider: openstack +# clusterStackName: scs +# kubernetesVersion: 1.34 # minor-only or with patch (1.34.3) +# +# Addon versions are read directly from cluster-addon/*/Chart.yaml as +# maintained by `just update addons`. The build does not resolve or +# modify addon versions. +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# OCI_REGISTRY OCI registry (default: ttl.sh) +# OCI_REPOSITORY OCI repository (auto-generated for ttl.sh) +# OCI_USERNAME OCI auth username (optional) +# OCI_PASSWORD OCI auth password (optional) +# OCI_ACCESS_TOKEN OCI auth token (optional, alternative to user/pass) +# OUTPUT_DIR Output directory (default: .release) +# CSO_CHART CSO Helm chart reference (default: oci://registry.scs.community/cluster-stacks/cso) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +YQ_VERSION="$(yq --version 2>/dev/null || true)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +yq_edit_in_place() { + local expression="$1" + local file="$2" + + if [[ "$YQ_VERSION" == *"https://github.com/mikefarah/yq/"* ]]; then + yq -i "$expression" "$file" + else + yq -y -i "$expression" "$file" + fi +} + +yq_has_path() { + local expression="$1" + local file="$2" + + yq -e "$expression" "$file" >/dev/null 2>&1 +} + +require_command yq +require_command curl +require_command jq +require_command helm + +# ============================================ +# Argument parsing +# ============================================ + +BASE_DIR="" +TARGET_VERSION="" +BUILD_ALL=false +PUBLISH=false +INSTALL_CSO=false +VALIDATE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) TARGET_VERSION="$2"; shift 2 ;; + --all) BUILD_ALL=true; shift ;; + --publish) PUBLISH=true; shift ;; + --install-cso) INSTALL_CSO=true; shift ;; + --validate) VALIDATE=true; shift ;; + -*) echo "Unknown option: $1"; exit 1 ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1"; exit 1 + fi + ;; + esac +done + +if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" +fi + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" + exit 1 +fi + +OUTPUT_DIR="${OUTPUT_DIR:-.release}" + +# ============================================ +# Resolve which minor-version directories to build +# ============================================ + +collect_version_dirs() { + if [[ "$BUILD_ALL" == "true" ]]; then + # All 1-* subdirectories, sorted + for d in "$BASE_DIR"/1-*/; do + [[ -d "$d" ]] && echo "$d" + done | sort -t- -k2 -n + elif [[ -n "$TARGET_VERSION" ]]; then + # --version 1.34 → look for 1-34/ + local minor_dash="${TARGET_VERSION//./-}" + local target_dir="$BASE_DIR/$minor_dash" + if [[ ! -d "$target_dir" ]]; then + echo "Version directory not found: $target_dir" >&2 + echo "Available:" >&2 + ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sed 's|.*/||; s|/$||; s/^/ /' >&2 + return 1 + fi + echo "$target_dir" + else + # Default: highest 1-* directory + local highest + highest=$(ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sort -t- -k2 -n | tail -1) + if [[ -z "$highest" ]]; then + echo "No version directories found in: $BASE_DIR" >&2 + return 1 + fi + echo "$highest" + fi +} + +VERSION_DIRS=$(collect_version_dirs) + +if [[ -z "$VERSION_DIRS" ]]; then + echo "No version directories to build" + exit 1 +fi + +# ============================================ +# OCI registry setup +# ============================================ + +setup_oci() { + if [[ -z "${OCI_REGISTRY:-}" ]]; then + DATE_YYYYMMDD="${OCI_DATE:-$(date +%Y%m%d)}" + export OCI_REGISTRY="ttl.sh" + export OCI_REPOSITORY="clusterstacks-${DATE_YYYYMMDD}" + echo "Auto-configured ttl.sh: $OCI_REGISTRY/$OCI_REPOSITORY (expires in 24h)" + fi +} + +OCI_SETUP_DONE=false +ensure_oci() { + if [[ "$OCI_SETUP_DONE" != "true" ]]; then + setup_oci + OCI_SETUP_DONE=true + fi +} + +# ============================================ +# CSO installation +# ============================================ + +CSO_CHART="${CSO_CHART:-oci://registry.scs.community/cluster-stacks/cso}" + +install_cso() { + if ! command -v helm >/dev/null 2>&1; then + echo "helm not found — install from https://helm.sh/docs/intro/install/" + exit 1 + fi + + echo "Installing/upgrading CSO..." + echo " Chart: $CSO_CHART" + echo " OCI config: $OCI_REGISTRY/$OCI_REPOSITORY" + echo "" + + helm upgrade -i cso "$CSO_CHART" \ + --namespace cso-system --create-namespace \ + --set controllerManager.manager.source=oci \ + --set "clusterStackVariables.ociRegistry=${OCI_REGISTRY}" \ + --set "clusterStackVariables.ociRepository=${OCI_REPOSITORY}" + + echo "" +} + +# Determine release version for a given K8s minor version. +# Stable: queries OCI for highest vN tag, returns v(N+1). Dev: returns v0-. +get_release_version() { + local provider="$1" + local stack_name="$2" + local k8s_dash="$3" + local tag_prefix="${provider}-${stack_name}-${k8s_dash}" + + if [[ "${OCI_REGISTRY:-}" == "ttl.sh" ]] || [[ -z "${OCI_REPOSITORY:-}" ]]; then + # Dev version + local timestamp + timestamp=$(date +%s) + echo "v0-ttl.${timestamp}" + return + fi + + local oras_opts=() + if [[ -n "${OCI_USERNAME:-}" && -n "${OCI_PASSWORD:-}" ]]; then + oras_opts+=(--username "$OCI_USERNAME" --password "$OCI_PASSWORD") + elif [[ -n "${OCI_ACCESS_TOKEN:-}" ]]; then + oras_opts+=(--password "$OCI_ACCESS_TOKEN") + fi + + # Query OCI for existing stable versions + require_command oras + + local latest=0 + local tags + if ! tags=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" "${oras_opts[@]}" 2>/dev/null); then + echo "Failed to query OCI tags from ${OCI_REGISTRY}/${OCI_REPOSITORY}." >&2 + exit 1 + fi + if [[ -n "$tags" ]]; then + latest=$(echo "$tags" | extract_latest_release_number "$tag_prefix" || echo "0") + latest="${latest:-0}" + fi + + echo "v$((latest + 1))" +} + +# ============================================ +# Resolve K8s patch version +# ============================================ + +# Given a version like "1.34" (no patch), resolve to latest patch. +# If already has patch (1.34.3), return as-is. +resolve_k8s_version() { + local version="$1" + local provider="$2" + + # Already has patch version + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + # Minor-only: resolve latest patch + if [[ "$provider" == "docker" ]]; then + # Query Docker Hub for kindest/node tags + local latest + if ! latest=$(curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&name=v${version}." 2>/dev/null \ + | jq -r '.results[].name' 2>/dev/null \ + | grep -E "^v${version}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1); then + echo "Failed to resolve the latest Docker patch version for Kubernetes ${version}." >&2 + exit 1 + fi + if [[ -n "$latest" ]]; then + echo "$latest" + return + fi + else + # Query GitHub for K8s releases + local github_headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + github_headers=(-H "Authorization: token $GITHUB_TOKEN") + fi + local latest + if ! latest=$(curl -sfL "${github_headers[@]+"${github_headers[@]}"}" \ + "https://api.github.com/repos/kubernetes/kubernetes/releases?per_page=100" 2>/dev/null \ + | jq -r '.[].tag_name' 2>/dev/null \ + | grep -E "^v${version}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1); then + echo "Failed to resolve the latest GitHub patch version for Kubernetes ${version}." >&2 + exit 1 + fi + if [[ -n "$latest" ]]; then + echo "$latest" + return + fi + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +# ============================================ +# Generate csctl.yaml for release artifact +# ============================================ + +generate_csctl_yaml() { + local provider="$1" + local stack_name="$2" + local k8s_version="$3" + local output_file="$4" + + cat > "$output_file" < /dev/null + echo " cluster-class packaged" + + # ---- Package cluster-addon bundle ---- + echo " Packaging cluster-addon..." + local addon_temp + addon_temp=$(mktemp -d) + local addon_count=0 + + for addon_dir in "$work_dir"/cluster-addon/*/; do + [[ -d "$addon_dir" ]] || continue + local addon_name + addon_name=$(basename "$addon_dir") + cp -r "$addon_dir" "$addon_temp/$addon_name" + rm -rf "$addon_temp/$addon_name/charts" + addon_count=$((addon_count + 1)) + done + + if [[ $addon_count -eq 0 ]]; then + echo " No addon subdirectories found" + rm -rf "$addon_temp" + exit 1 + fi + + local addon_tgz="${provider}-${stack_name}-${k8s_dash}-cluster-addon-${release_version}.tgz" + (cd "$addon_temp" && tar -czf "$(cd "$REPO_ROOT" && pwd)/$release_dir/$addon_tgz" */) + rm -rf "$addon_temp" + echo " cluster-addon packaged ($addon_count addons)" + + # ---- Validate addon bundle ---- + if [[ "$VALIDATE" == "true" && -f "$version_dir/clusteraddon.yaml" ]]; then + echo " Validating addon bundle..." + local validate_dir + validate_dir=$(mktemp -d) + tar -xzf "$release_dir/$addon_tgz" -C "$validate_dir" + + local expected_addons + expected_addons=$(yq '.addonStages | to_entries | .[].value[].name' "$version_dir/clusteraddon.yaml" 2>/dev/null | sort -u) + + local failed=false + for addon in $expected_addons; do + if [[ ! -d "$validate_dir/$addon" ]]; then + echo " Missing addon: $addon (referenced in clusteraddon.yaml)" + failed=true + fi + done + rm -rf "$validate_dir" + + if [[ "$failed" == "true" ]]; then + exit 1 + fi + echo " Validation passed" + fi + + # ---- Copy clusteraddon.yaml ---- + if [[ -f "$version_dir/clusteraddon.yaml" ]]; then + cp "$version_dir/clusteraddon.yaml" "$release_dir/" + fi + + # ---- Copy generated csctl.yaml ---- + cp "$work_dir/csctl.yaml" "$release_dir/" + + # ---- Generate metadata.yaml ---- + local git_hash + git_hash=$(git rev-parse HEAD 2>/dev/null || echo "unknown") + + cat > "$release_dir/metadata.yaml" < "$release_dir/hashes.json" </dev/null 2>&1; then + echo " oras not found — install from https://oras.land/docs/installation" + exit 1 + fi + + echo "" + echo " Publishing to $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" + + local oras_opts=() + if [[ -n "${OCI_USERNAME:-}" && -n "${OCI_PASSWORD:-}" ]]; then + oras_opts+=(--username "$OCI_USERNAME" --password "$OCI_PASSWORD") + elif [[ -n "${OCI_ACCESS_TOKEN:-}" ]]; then + oras_opts+=(--password "$OCI_ACCESS_TOKEN") + fi + + local files=() + for f in "$release_dir"/*; do + [[ -f "$f" ]] && files+=("$(basename "$f")") + done + + (cd "$release_dir" && oras push \ + "$OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" \ + --artifact-type application/vnd.clusterstack.release \ + "${oras_opts[@]}" \ + "${files[@]}") + + echo " Published: $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" + echo " Pull: oras pull $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" +} + +# ============================================ +# Main +# ============================================ + +# Resolve OCI config once (needed for --publish and/or --install-cso) +if [[ "$PUBLISH" == "true" || "$INSTALL_CSO" == "true" ]]; then + ensure_oci +fi + +# Install/upgrade CSO if requested +if [[ "$INSTALL_CSO" == "true" ]]; then + install_cso +fi + +echo "Stack base: $BASE_DIR" +echo "Version dirs: $(echo "$VERSION_DIRS" | tr '\n' ' ')" +echo "" + +# Track built versions for "next steps" output +declare -a BUILT_K8S_SHORTS=() +declare -a BUILT_CS_VERSIONS=() +declare -a BUILT_PROVIDERS=() +declare -a BUILT_STACK_NAMES=() + +for version_dir in $VERSION_DIRS; do + build_version_dir "$version_dir" +done + +echo "" +echo "Done." + +# ============================================ +# Next steps (after publish) +# ============================================ + +if [[ "$PUBLISH" == "true" && ${#BUILT_K8S_SHORTS[@]} -gt 0 ]]; then + echo "" + echo "================================================================" + echo "Next steps" + echo "================================================================" + + if [[ "$INSTALL_CSO" != "true" ]]; then + echo "" + echo "1. Install the Cluster Stack Operator (or re-run with --install-cso):" + echo "" + echo " helm upgrade -i cso ${CSO_CHART} \\" + echo " --namespace cso-system --create-namespace \\" + echo " --set controllerManager.manager.source=oci \\" + echo " --set clusterStackVariables.ociRegistry=\"${OCI_REGISTRY}\" \\" + echo " --set clusterStackVariables.ociRepository=\"${OCI_REPOSITORY}\"" + echo "" + echo "2. Apply the ClusterStack resource(s):" + else + echo "" + echo "Apply the ClusterStack resource(s):" + fi + echo "" + for ((i=0; i<${#BUILT_K8S_SHORTS[@]}; i++)); do + local_provider="${BUILT_PROVIDERS[$i]}" + local_stack="${BUILT_STACK_NAMES[$i]}" + local_k8s="${BUILT_K8S_SHORTS[$i]}" + local_version="${BUILT_CS_VERSIONS[$i]}" + echo " CLUSTER_STACK=${local_stack} ./hack/generate-resources.sh --version ${local_k8s} --cs-version ${local_version} | kubectl apply -f -" + done + echo "" +fi diff --git a/hack/config-template.md b/hack/config-template.md index 8f05340b..091ee975 100644 --- a/hack/config-template.md +++ b/hack/config-template.md @@ -2,14 +2,18 @@ This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. +## Version matrix + +!!matrix!! + ## Example ```yaml -apiVersion: cluster.x-k8s.io/v1beta1 +apiVersion: cluster.x-k8s.io/v1beta2 kind: Cluster metadata: - name: - namespace: + name: my-cluster + namespace: my-namespace labels: managed-secret: cloud-config spec: @@ -22,38 +26,41 @@ spec: cidrBlocks: - 10.96.0.0/12 topology: - variables: // <-- variables from the table can be set here - - name: controller_flavor + variables: # <-- variables from the table below can be set here + - name: flavor value: "SCS-4V-8-20" - - name: worker_flavor - value: "SCS-4V-8-20" - - name: external_id + - name: networkExternalID value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" - class: openstack-alpha-1-29-v2 + class: openstack-scs-1-35-v1 controlPlane: - replicas: 2 - version: v1.29.3 + replicas: 3 + variables: + overrides: + - name: flavor + value: "SCS-4V-8-50" + version: v1.35.0 workers: machineDeployments: - - class: openstack-alpha-1-29-v2 - failureDomain: nova - name: openstack-alpha-1-29-v2 - replicas: 4 + - class: default-worker + name: md-0 + replicas: 3 ``` -Variables from the table containing a `.` are to be used in an object with the part before the dot being the object name and the part behind the dot being the value names. The following example demonstrates this with `oidc_config`. +Variables of type `object` are set as nested values. The following example demonstrates this with `oidcConfig`: ```yaml ... topology: variables: - - name: oidc_config + - name: oidcConfig value: - issuer_url: "https://dex.k8s.scs.community" - client_id: "kubectl" + issuerURL: "https://dex.k8s.scs.community" + clientID: "kubectl" ... ``` +In v1beta2, per-role overrides (e.g. different flavors for control plane and workers) are set via `topology.controlPlane.variables.overrides` and `topology.workers.machineDeployments[].variables.overrides` instead of separate variable names. + ## Available variables -!!table!! \ No newline at end of file +!!table!! diff --git a/hack/docugen.py b/hack/docugen.py index 1f58fdd7..097bea33 100755 --- a/hack/docugen.py +++ b/hack/docugen.py @@ -1,149 +1,171 @@ #!/usr/bin/env python3 """ -Generate markdown table from cluster-class variable definitions. +Generate markdown documentation from ClusterClass variable definitions. -This script temporarily renders the helm template of the cluster-class. -Parses the `Cluster.spec.topology.variables` definitions and -generates a markdown table for documentation purposes. +Renders the cluster-class Helm template, parses the topology variables +(openAPIV3Schema), and outputs a markdown table of all configurable options. + +Usage: + ./hack/docugen.py + ./hack/docugen.py --output docs/configuration.md + ./hack/docugen.py --template hack/config-template.md + ./hack/docugen.py --matrix + ./hack/docugen.py --dry-run """ import argparse import subprocess -from pathlib import Path import sys +from pathlib import Path import yaml -BASE_PATH = Path(__file__).parent.parent -TEMPLATE_PATH = BASE_PATH.joinpath("providers", "openstack", "scs", "cluster-class") -DOCS_TMPL_PATH = BASE_PATH.joinpath("hack", "config-template.md") -DOCS_OUT_PATH = BASE_PATH.joinpath("docs", "providers", "openstack", "configuration.md") - - -def generate_row(content: list): - """Generate string of markdown table row.""" - row_tmpl = "|{content}|" - row_str = "|".join(content) - - return row_tmpl.format(content=row_str) - - -def parse_variable(tmpl: dict) -> list: - """ - Parse schema of simple cluster-stack variable type. (String, Boolean, Integer, Array) - Parameters: - tmpl (dict): Dictionary of variable schema, - parsed from cluster-class.yaml via yaml.safe_load(). +def generate_row(columns: list) -> str: + """Generate a markdown table row from a list of column values.""" + return "|" + "|".join(str(c) for c in columns) + "|" - Returns: List of variable schema properties. - Each entry represents a column in the final markdown table row. - """ - var_name = tmpl["name"] - var_required = tmpl["required"] - var_schema = tmpl["schema"]["openAPIV3Schema"] - var_type = var_schema["type"] - var_default = var_schema.get("default", "") - var_example = var_schema.get("example", "") - var_desc = var_schema.get("description", "TODO") +def parse_variable(var: dict) -> list: + """Parse a simple variable (string, boolean, integer, array) into table columns.""" + name = var["name"] + required = var["required"] + schema = var["schema"]["openAPIV3Schema"] - row = [] - row.append(f"`{var_name}`") - row.append(var_type) + var_type = schema["type"] + default = schema.get("default", "") + example = schema.get("example", "") + description = schema.get("description", "TODO").replace("\n", "
") - # Make sure that example and default values are quoted in the table if var_type == "string": - row.append(f'"{var_default}"') - row.append(f'"{var_example}"') - - # Cast any non-string type example / default to string - if var_type in ["array", "integer", "boolean"]: - row.append(str(var_default)) - row.append(str(var_example)) - - row.append(var_desc.replace("\n", "
")) - row.append(str(var_required)) # Convert boolean variable to string - - return row - - -def parse_object(tmpl: dict) -> list: - """ - Parse cluster-stack variable schema of type object. - Separated due to nesting in YAML format. - Parameters: - tmpl (dict): Dictionary of object schema, - parsed from cluster-class.yaml via yaml.safe_load(). - - Returns: - List of object properties. - Each entry represents a row in the final markdown table as a list, - consisting of the rows column values. - """ - var_name = tmpl["name"] - props = tmpl["schema"]["openAPIV3Schema"]["properties"] - - object_list = [] - for prop in props: - row = [] - row.append(f"`{var_name}.{prop}`") - row.append(props[prop]["type"]) - row.append(props[prop].get("default", "")) - row.append(props[prop].get("example", "")) - row.append(props[prop].get("description", "TODO")) - # append dummy row for required field - row.append("") - - object_list.append(row) - return object_list + default = f'"{default}"' + example = f'"{example}"' + else: + default = str(default) + example = str(example) + + return [f"`{name}`", var_type, default, example, description, str(required)] + + +def parse_object(var: dict) -> list: + """Parse an object variable into multiple table rows (one per property).""" + name = var["name"] + props = var["schema"]["openAPIV3Schema"]["properties"] + rows = [] + + for prop_name, prop_schema in props.items(): + rows.append([ + f"`{name}.{prop_name}`", + prop_schema["type"], + prop_schema.get("default", ""), + prop_schema.get("example", ""), + prop_schema.get("description", "TODO"), + "", + ]) + + return rows + + +def render_cluster_class(stack_dir: Path) -> dict: + """Render the cluster-class Helm template and return parsed YAML.""" + cluster_class_dir = stack_dir / "cluster-class" + if not cluster_class_dir.exists(): + print(f"cluster-class directory not found: {cluster_class_dir}", file=sys.stderr) + sys.exit(1) + + result = subprocess.run( + ["helm", "template", "docugen", str(cluster_class_dir), + "-s", "templates/cluster-class.yaml"], + capture_output=True, check=False, + ) + if result.returncode != 0: + print(f"helm template failed:\n{result.stderr.decode()}", file=sys.stderr) + sys.exit(1) -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--dry-run", action="store_true", help="Only print result to stdout." - ) + return yaml.safe_load(result.stdout.decode("utf-8")) - args = parser.parse_args() - cmd = [ - "helm", - "template", - "docugen", - TEMPLATE_PATH, - "-s", - "templates/cluster-class.yaml", +def generate_table(template: dict) -> str: + """Generate a markdown table from ClusterClass topology variables.""" + rows = [ + "|Name|Type|Default|Example|Description|Required|", + "|----|----|-------|-------|-----------|--------|", ] - with open(DOCS_TMPL_PATH, "r") as f: - tmpl = f.read() + for var in template["spec"]["variables"]: + var_type = var["schema"]["openAPIV3Schema"]["type"] + if var_type == "object": + for row in parse_object(var): + rows.append(generate_row(row)) + else: + rows.append(generate_row(parse_variable(var))) - cmdout = subprocess.run(cmd, capture_output=True, check=False) - rendered_template = cmdout.stdout.decode("utf-8") + return "\n".join(rows) - template = yaml.safe_load(rendered_template) - result_table = [] - result_table.append("|Name|Type|Default|Example|Description|Required|") - result_table.append("|----|----|-------|-------|-----------|--------|") +def main(): + parser = argparse.ArgumentParser( + description="Generate docs from ClusterClass variables", + ) + parser.add_argument( + "stack_dir", type=Path, + help="Path to the cluster stack directory", + ) + parser.add_argument( + "--template", type=Path, default=None, + help="Markdown template file with !!table!! placeholder", + ) + parser.add_argument( + "--output", "-o", type=Path, default=None, + help="Output file path (default: stdout)", + ) + parser.add_argument( + "--matrix", action="store_true", + help="Include version matrix from show-matrix.sh --markdown", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="Print to stdout even if --output is set", + ) + args = parser.parse_args() - for var in template["spec"]["variables"]: - if var["schema"]["openAPIV3Schema"]["type"] in ["object"]: - parsed = parse_object(var) - for object_property in parsed: - result_table.append(generate_row(object_property)) + # Render and parse + template = render_cluster_class(args.stack_dir) + table = generate_table(template) + + # Generate version matrix if requested + matrix = "" + if args.matrix: + # Derive the stack base directory (parent of the version dir) + stack_base = args.stack_dir.parent + script = Path(__file__).parent / "show-matrix.sh" + result = subprocess.run( + ["bash", str(script), "--markdown", str(stack_base)], + capture_output=True, check=False, + ) + if result.returncode == 0: + matrix = result.stdout.decode("utf-8").strip() else: - parsed = parse_variable(var) - result_table.append(generate_row(parsed)) - - output = tmpl.replace("!!table!!", "\n".join(result_table)) - - if args.dry_run: + print(f"show-matrix.sh failed:\n{result.stderr.decode()}", file=sys.stderr) + + # Apply template if provided + if args.template and args.template.exists(): + output = args.template.read_text() + output = output.replace("!!table!!", table) + output = output.replace("!!matrix!!", matrix) + else: + output = table + + # Output + if args.dry_run or args.output is None: print(output) - sys.exit() + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output) + print(f"Written: {args.output}", file=sys.stderr) - print(f"Writing output to file {DOCS_OUT_PATH}") - with open(DOCS_OUT_PATH, "w") as f: - f.write(output) + +if __name__ == "__main__": + main() diff --git a/hack/ensure-connected-to-mgt-cluster.sh b/hack/ensure-connected-to-mgt-cluster.sh deleted file mode 100755 index 5eda91e7..00000000 --- a/hack/ensure-connected-to-mgt-cluster.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -context="$(kubectl config current-context 2>/dev/null || true)" - -if [ "$context" = "kind-scs-cluster-stacks" ]; then - exit 0 -fi - -if [ "$context" = "" ]; then - echo "No context set" - exit 1 -fi - - -echo "You are connected to $context. Please set KUBECONFIG to .mgt-cluster-kubeconfig.yaml" -exit 1 - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 VAR1 VAR2 ..." - exit 1 -fi - -missing_vars=() -for varname in "$@"; do - eval varvalue="\$$varname" - if [ -z "$varvalue" ]; then - missing_vars+=("$varname") - fi -done - -if [ ${#missing_vars[@]} -gt 0 ]; then - echo "Missing or empty environment variables: ${missing_vars[*]}" - exit 1 -fi diff --git a/hack/ensure-env-variables.sh b/hack/ensure-env-variables.sh deleted file mode 100755 index 8dc027e4..00000000 --- a/hack/ensure-env-variables.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 VAR1 VAR2 ..." - exit 1 -fi - -missing_vars=() -for varname in "$@"; do - eval varvalue="\$$varname" - if [ -z "$varvalue" ]; then - missing_vars+=("$varname") - fi -done - -if [ ${#missing_vars[@]} -gt 0 ]; then - echo "Missing or empty environment variables: ${missing_vars[*]}" - exit 1 -fi diff --git a/hack/generate-image-manifests.sh b/hack/generate-image-manifests.sh new file mode 100755 index 00000000..6e095562 --- /dev/null +++ b/hack/generate-image-manifests.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +# Generate OpenStack image manifests for Kubernetes CAPI images. +# +# Supports two output formats: +# orc ORC Image CRD (default) — for k-orc.cloud Image resources +# image-manager openstack-image-manager YAML — for OSISM image-manager +# +# Usage: +# ./hack/generate-image-manifests.sh [stack-dir] [options] +# +# Options: +# --version Generate for a specific K8s minor version +# --format Output format: orc (default) or image-manager +# --visibility Image visibility: private (default), public, shared, community +# --output-dir Write to files instead of stdout +# --skip-checksum Skip fetching SHA256 checksums +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). +# +# Only relevant for OpenStack-based stacks (Docker stacks have no node images). +# +# Examples: +# ./hack/generate-image-manifests.sh --version 1.34 +# ./hack/generate-image-manifests.sh --format image-manager --visibility public +# ./hack/generate-image-manifests.sh --version 1.34 --visibility shared --skip-checksum +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# IMAGE_BASE_URL Base URL for images (default: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images) +# CLOUD_NAME CloudCredentialsRef cloud name (default: openstack, orc format only) +# SECRET_NAME CloudCredentialsRef secret name (default: openstack, orc format only) + +set -euo pipefail + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +require_command yq +require_command curl +require_command jq + +# Defaults +BASE_URL="${IMAGE_BASE_URL:-https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images}" +CLOUD_NAME="${CLOUD_NAME:-openstack}" +SECRET_NAME="${SECRET_NAME:-openstack}" + +# ============================================ +# Argument parsing +# ============================================ + +BASE_DIR="" +TARGET_VERSION="" +OUTPUT_DIR="" +OUTPUT_FORMAT="orc" +VISIBILITY="private" +SKIP_CHECKSUM=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) TARGET_VERSION="$2"; shift 2 ;; + --format) OUTPUT_FORMAT="$2"; shift 2 ;; + --visibility) VISIBILITY="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + --skip-checksum) SKIP_CHECKSUM=true; shift ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; exit 1 + fi + ;; + esac +done + +if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" +fi + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 +fi + +case "$OUTPUT_FORMAT" in + orc|image-manager) ;; + *) echo "Unknown format: $OUTPUT_FORMAT (use 'orc' or 'image-manager')" >&2; exit 1 ;; +esac + +case "$VISIBILITY" in + private|public|shared|community) ;; + *) echo "Unknown visibility: $VISIBILITY (use private, public, shared, or community)" >&2; exit 1 ;; +esac + +# ============================================ +# Ubuntu version mapping +# ============================================ + +# K8s 1.32 and earlier → Ubuntu 22.04 (2204) +# K8s 1.33 and later → Ubuntu 24.04 (2404) +ubuntu_for_minor() { + local minor="$1" + if [[ "$minor" -le 32 ]]; then + echo "2204" + else + echo "2404" + fi +} + +# ============================================ +# Check provider +# ============================================ + +FIRST_STACK="" +for stack_file in "$BASE_DIR"/1-*/stack.yaml; do + if [[ -f "$stack_file" ]]; then + FIRST_STACK="$stack_file" + break + fi +done + +if [[ -z "$FIRST_STACK" ]]; then + echo "No stack.yaml found in $BASE_DIR/1-*/" >&2 + exit 1 +fi + +STACK_PROVIDER=$(yq -r '.provider' "$FIRST_STACK") +if [[ "$STACK_PROVIDER" != "openstack" ]]; then + echo "Image manifests are only relevant for OpenStack-based stacks (provider: $STACK_PROVIDER)." >&2 + exit 0 +fi + +[[ -n "$OUTPUT_DIR" ]] && mkdir -p "$OUTPUT_DIR" + +# ============================================ +# Resolve K8s patch version +# ============================================ + +resolve_k8s_version() { + local version="$1" + + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + local github_headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + github_headers=(-H "Authorization: token $GITHUB_TOKEN") + fi + local latest + if ! latest=$(curl -sfL "${github_headers[@]+"${github_headers[@]}"}" \ + "https://api.github.com/repos/kubernetes/kubernetes/releases?per_page=100" 2>/dev/null \ + | jq -r '.[].tag_name' 2>/dev/null \ + | grep -E "^v${version}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1); then + echo "Failed to resolve the latest GitHub patch version for Kubernetes ${version}." >&2 + exit 1 + fi + + if [[ -n "$latest" ]]; then + echo "$latest" + return + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +# ============================================ +# Collect image data +# ============================================ + +# Arrays to collect data for image-manager format (needs all versions grouped) +declare -a ALL_K8S_VERSIONS=() +declare -a ALL_IMAGE_URLS=() +declare -a ALL_CHECKSUMS=() +declare -a ALL_UBUNTU=() + +GENERATED=0 +FAIL_COUNT=0 + +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + stack_yaml="$version_dir/stack.yaml" + [[ -f "$stack_yaml" ]] || continue + + k8s_version_raw=$(yq -r '.kubernetesVersion' "$stack_yaml") + k8s_short=$(extract_k8s_minor_version "$k8s_version_raw") + k8s_minor=$(echo "$k8s_short" | cut -d. -f2) + + # Filter by --version if specified + if [[ -n "$TARGET_VERSION" ]]; then + target_short=$(extract_k8s_minor_version "$TARGET_VERSION") + if [[ "$k8s_short" != "$target_short" ]]; then + continue + fi + fi + + # Resolve full K8s patch version + k8s_version=$(resolve_k8s_version "$k8s_version_raw") + + UBUNTU=$(ubuntu_for_minor "$k8s_minor") + IMAGE_NAME="ubuntu-${UBUNTU}-kube-v${k8s_version}" + IMAGE_DIR="ubuntu-${UBUNTU}-kube-v${k8s_short}" + IMAGE_URL="${BASE_URL}/${IMAGE_DIR}/${IMAGE_NAME}.qcow2" + + # Fetch checksum + CHECKSUM="" + if [[ "$SKIP_CHECKSUM" != "true" ]]; then + CHECKSUM=$(curl -sf "${IMAGE_URL}.CHECKSUM" | awk '{print $1}' || echo "") + if [[ -z "$CHECKSUM" ]]; then + echo "Failed to fetch checksum for ${k8s_version}: ${IMAGE_URL}.CHECKSUM" >&2 + echo "Use --skip-checksum to generate without hash validation" >&2 + FAIL_COUNT=$((FAIL_COUNT + 1)) + continue + fi + fi + + ALL_K8S_VERSIONS+=("$k8s_version") + ALL_IMAGE_URLS+=("$IMAGE_URL") + ALL_CHECKSUMS+=("$CHECKSUM") + ALL_UBUNTU+=("$UBUNTU") + GENERATED=$((GENERATED + 1)) +done + +# ============================================ +# Output: ORC Image CRD format +# ============================================ + +generate_orc() { + for ((i=0; i<${#ALL_K8S_VERSIONS[@]}; i++)); do + local k8s_version="${ALL_K8S_VERSIONS[$i]}" + local image_url="${ALL_IMAGE_URLS[$i]}" + local checksum="${ALL_CHECKSUMS[$i]}" + local ubuntu="${ALL_UBUNTU[$i]}" + + local hash_block="" + if [[ -n "$checksum" ]]; then + hash_block=" + hash: + algorithm: sha256 + value: ${checksum}" + fi + + local manifest="--- +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Image +metadata: + name: ubuntu-capi-image-v${k8s_version} +spec: + cloudCredentialsRef: + cloudName: ${CLOUD_NAME} + secretName: ${SECRET_NAME} + managementPolicy: managed + resource: + visibility: ${VISIBILITY} + properties: + hardware: + diskBus: scsi + scsiModel: virtio-scsi + vifModel: virtio + qemuGuestAgent: true + rngModel: virtio + architecture: x86_64 + minDiskGB: 20 + minMemoryMB: 2048 + operatingSystem: + distro: ubuntu + version: \"${ubuntu:0:2}.${ubuntu:2:2}\" + content: + diskFormat: qcow2 + download: + url: ${image_url}${hash_block}" + + if [[ -n "$OUTPUT_DIR" ]]; then + local outfile="${OUTPUT_DIR}/ubuntu-${ubuntu}-kube-v${k8s_version}.yaml" + echo "$manifest" > "$outfile" + echo "Written: $outfile" >&2 + else + echo "$manifest" + fi + done +} + +# ============================================ +# Output: openstack-image-manager format +# ============================================ + +generate_image_manager() { + # Build versions list + local versions_block="" + for ((i=0; i<${#ALL_K8S_VERSIONS[@]}; i++)); do + local k8s_version="${ALL_K8S_VERSIONS[$i]}" + local image_url="${ALL_IMAGE_URLS[$i]}" + local checksum="${ALL_CHECKSUMS[$i]}" + + local checksum_line="" + if [[ -n "$checksum" ]]; then + checksum_line=" + checksum: \"sha256:${checksum}\"" + fi + + versions_block+=" + - version: 'v${k8s_version}' + url: ${image_url}${checksum_line}" + done + + local manifest="--- +images: + - name: ubuntu-capi-image + enable: true + format: raw + login: ubuntu + min_disk: 20 + min_ram: 1024 + status: active + visibility: ${VISIBILITY} + multi: false + separator: \"-\" + meta: + architecture: x86_64 + hw_disk_bus: virtio + hw_rng_model: virtio + hw_scsi_model: virtio-scsi + hw_watchdog_action: reset + hypervisor_type: qemu + os_distro: ubuntu + os_purpose: k8snode + replace_frequency: never + uuid_validity: none + provided_until: none + tags: + - clusterstacks + versions:${versions_block}" + + if [[ -n "$OUTPUT_DIR" ]]; then + local outfile="${OUTPUT_DIR}/kubernetes.yaml" + echo "$manifest" > "$outfile" + echo "Written: $outfile" >&2 + else + echo "$manifest" + fi +} + +# ============================================ +# Generate output +# ============================================ + +if [[ $GENERATED -eq 0 && $FAIL_COUNT -eq 0 ]]; then + echo "No matching versions found" >&2 + exit 1 +fi + +if [[ $FAIL_COUNT -gt 0 && $GENERATED -eq 0 ]]; then + echo "All $FAIL_COUNT version(s) failed checksum fetch" >&2 + exit 1 +fi + +case "$OUTPUT_FORMAT" in + orc) generate_orc ;; + image-manager) generate_image_manager ;; +esac + +if [[ $FAIL_COUNT -gt 0 ]]; then + echo "Generated $GENERATED manifest(s), failed $FAIL_COUNT" >&2 +else + echo "Generated $GENERATED manifest(s)" >&2 +fi diff --git a/hack/generate-resources.sh b/hack/generate-resources.sh new file mode 100755 index 00000000..57a4f29f --- /dev/null +++ b/hack/generate-resources.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Generate ClusterStack and Cluster YAML resources for testing. +# +# Usage: +# ./hack/generate-resources.sh [stack-dir] --version 1.34 [options] +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). +# +# Options: +# --version K8s minor version (required) +# --cs-version Cluster stack version (default: auto-detect, see below) +# --namespace Namespace (default: cluster) +# --cluster-name Workload cluster name (default: cs-cluster) +# --cluster-only Only generate the Cluster resource +# --clusterstack-only Only generate the ClusterStack resource +# +# Auto-detection of --cs-version (in priority order): +# 1. Local .release/ directory — reads metadata.yaml from the latest matching build +# 2. OCI registry — queries tags via oras (requires OCI_REGISTRY + OCI_REPOSITORY) +# 3. Falls back to v1 +# +# Output goes to stdout. Pipe to kubectl apply -f - or redirect to a file. +# +# Examples: +# ./hack/generate-resources.sh --version 1.34 +# ./hack/generate-resources.sh --version 1.34 --cs-version v2 +# ./hack/generate-resources.sh --version 1.34 | kubectl apply -f - +# PROVIDER=docker ./hack/generate-resources.sh --version 1.35 + +set -euo pipefail + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +require_command yq +require_command curl +require_command jq + +resolve_k8s_version() { + local version="$1" + local provider="$2" + + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + if [[ "$provider" == "docker" ]]; then + local latest_docker + if ! latest_docker=$(curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&name=v${version}." 2>/dev/null | \ + jq -r '.results[].name' 2>/dev/null | \ + grep -E "^v${version}\.[0-9]+$" | \ + sed 's/^v//' | \ + sort -V | \ + tail -1); then + echo "Failed to resolve the latest Docker patch version for Kubernetes ${version}." >&2 + exit 1 + fi + if [[ -n "$latest_docker" ]]; then + echo "$latest_docker" + return + fi + else + local github_headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + github_headers=(-H "Authorization: token $GITHUB_TOKEN") + fi + local latest_github + if ! latest_github=$(curl -sfL "${github_headers[@]+"${github_headers[@]}"}" \ + "https://api.github.com/repos/kubernetes/kubernetes/releases?per_page=100" 2>/dev/null | \ + jq -r '.[].tag_name' 2>/dev/null | \ + grep -E "^v${version}\.[0-9]+$" | \ + sed 's/^v//' | \ + sort -V | \ + tail -1); then + echo "Failed to resolve the latest GitHub patch version for Kubernetes ${version}." >&2 + exit 1 + fi + if [[ -n "$latest_github" ]]; then + echo "$latest_github" + return + fi + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +# ============================================ +# Argument parsing +# ============================================ + +BASE_DIR="" +K8S_VERSION="" +CS_VERSION="" +NAMESPACE="cluster" +CLUSTER_NAME="cs-cluster" +CLUSTER_ONLY=false +CLUSTERSTACK_ONLY=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) K8S_VERSION="$2"; shift 2 ;; + --cs-version) CS_VERSION="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + --cluster-name) CLUSTER_NAME="$2"; shift 2 ;; + --cluster-only) CLUSTER_ONLY=true; shift ;; + --clusterstack-only) CLUSTERSTACK_ONLY=true; shift ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; exit 1 + fi + ;; + esac +done + +if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" +fi + +if [[ -z "$K8S_VERSION" ]]; then + echo "Usage: $0 [stack-dir] --version X.Y [--cs-version vN] [--namespace ns] [--cluster-name name]" >&2 + exit 1 +fi + +# Resolve version directory +K8S_DASH="${K8S_VERSION//./-}" +VERSION_DIR="$BASE_DIR/$K8S_DASH" + +if [[ ! -d "$VERSION_DIR" ]]; then + echo "Version directory not found: $VERSION_DIR" >&2 + echo "Available:" >&2 + ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sed 's|.*/||; s|/$||; s/^/ /' >&2 + exit 1 +fi + +STACK_YAML="$VERSION_DIR/stack.yaml" +if [[ ! -f "$STACK_YAML" ]]; then + echo "stack.yaml not found in: $VERSION_DIR" >&2 + exit 1 +fi + +# ============================================ +# Read stack configuration +# ============================================ + +PROVIDER=$(yq -r '.provider' "$STACK_YAML") +CLUSTER_STACK=$(yq -r '.clusterStackName' "$STACK_YAML") +K8S_VERSION_RAW=$(yq -r '.kubernetesVersion' "$STACK_YAML") + +K8S_FULL=$(resolve_k8s_version "$K8S_VERSION_RAW" "$PROVIDER") + +# Auto-detect CS version (if not specified) +# Priority: 1. local .release/ build output 2. OCI registry 3. fall back to v1 +if [[ -z "$CS_VERSION" ]]; then + REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + RELEASE_DIR="${OUTPUT_DIR:-$REPO_ROOT/.release}" + TAG_PREFIX="${PROVIDER}-${CLUSTER_STACK}-${K8S_DASH}" + + # 1. Check .release/ for a matching build (newest first by mtime) + LATEST_BUILD="" + if [[ -d "$RELEASE_DIR" ]]; then + LATEST_BUILD=$(ls -dt "$RELEASE_DIR/${TAG_PREFIX}"-*/metadata.yaml 2>/dev/null | head -1 || true) + fi + + if [[ -n "$LATEST_BUILD" ]]; then + CS_VERSION=$(yq -r '.versions.clusterStack' "$LATEST_BUILD") + echo "# Auto-detected CS version: ${CS_VERSION} (from $(dirname "$LATEST_BUILD"))" >&2 + # 2. Try OCI registry + elif [[ -n "${OCI_REGISTRY:-}" && -n "${OCI_REPOSITORY:-}" ]] && command -v oras >/dev/null 2>&1; then + LATEST=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" 2>/dev/null | \ + extract_latest_release_number "$TAG_PREFIX" || echo "") + if [[ -n "$LATEST" ]]; then + CS_VERSION="v${LATEST}" + echo "# Auto-detected CS version: ${CS_VERSION} (from ${OCI_REGISTRY}/${OCI_REPOSITORY})" >&2 + else + CS_VERSION="v1" + echo "# No published versions found, using default: ${CS_VERSION}" >&2 + fi + else + CS_VERSION="v1" + echo "# No .release/ build or OCI registry available, using default: ${CS_VERSION}" >&2 + fi +fi + +# ============================================ +# Generate ClusterStack resource +# ============================================ + +if [[ "$CLUSTER_ONLY" != "true" ]]; then + cat < list: - """ - Read supported versions from file for output or further usage inside this script. - - Parameters: - None - - Returns: - List of supported versions - """ - logger.info("Loading supported versions.") - version_file = SOURCE_PATH.joinpath("versions.yaml") - with open(version_file, encoding="utf-8") as stream: - try: - result = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - return result - - -def get_dash_version(version: str) -> str: - """ - Helper function to convert version from dotted to separated by hyphen. (1.27.14 -> 1-27) - Parameters: - version (string): String containing the full semver version. - - Returns: - String with shortened version separated by a hypgen. - """ - return "-".join(version.split(".")[0:2]) - - -def create_output_dir(version: str) -> PosixPath: - """ - Prepare output directory by creating it and copying the source files over. - This overwrites files existing inside the output directory, as those - should not be edited manually. - - Parameters: - version (string): Semver version string as read from the supported versions list. - - Returns: - PosixPath object of the created directory. - """ - out = get_dash_version(version) - out_dir = DEFAULT_TARGET_PATH.joinpath(out) - - logger.info("Creating output directory at %s", out_dir) - - # TODO: how to handle FileExistsError? - # as the output is being generated, it *should* be safe to overwrite - if out_dir.exists(): - shutil.rmtree(str(out_dir)) - - # Copy whole tree from src dir and remove "versions.yaml" file - shutil.copytree(SOURCE_PATH, out_dir) - out_dir.joinpath("versions.yaml").unlink() - for file in out_dir.joinpath("cluster-addon").rglob("Chart.lock"): - file.unlink() - for folder in out_dir.joinpath("cluster-addon").rglob("charts"): - shutil.rmtree(folder) - - return out_dir - - -def readfile(path: PosixPath): - """ - Helper function to read yaml configuration files. - - Parameters: - path (PosixPath): pathlib object of the file to open. - - Returns: - Content of the yaml configuration file. - """ - # TODO: yaml.safe_load either returns a list or dict, - # depending on the structure of the yaml file. This can be improved / refactored. - with open(path, encoding="utf-8") as stream: - try: - content = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - return content - - -def writefile(path: PosixPath, content): - """ - Helper function to write content to a yaml configuration file. - - Parameters: - path (PosixPath): pathlib object of the target file path. - content: yaml data to be written. This can either be of type list or dict. - - Returns: - None - """ - with open(path, "w", encoding="utf-8") as stream: - yaml.safe_dump(content, stream) - - -def update_cluster_addon( - target: PosixPath, build: bool, build_verbose: bool, **versions -): - """ - Update relevant files inside the cluster-stacks//cluster-addon subdirectory - - Parameters: - target (PosixPath): pathlib object of the relevant file. - build (boolean): Toggle to control if helm dependencies should be build, - build_verbose (boolean): Toggle to control if build output should be printed. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - for dep in content["dependencies"]: - if dep["name"] == "openstack-cinder-csi": - dep["version"] = versions["cinder_csi"] - - if dep["name"] == "openstack-cloud-controller-manager": - dep["version"] = versions["occm"] - - content["name"] = ( - f"openstack-scs-{get_dash_version(versions['kubernetes'])}-cluster-addon" - ) - - writefile(target, content) - - if build: - logger.info("Building helm dependencies") - cmd = ["helm", "dependency", "build"] - subprocess.run( - cmd, - cwd=str(target).replace("Chart.yaml", ""), - capture_output=build_verbose, - check=False, - ) - - -def update_csctl_conf(target: PosixPath, **versions): - """ - Function to update csctl configuration file. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - content["config"]["kubernetesVersion"] = f"v{versions['kubernetes']}" - - writefile(target, content) - - -def update_cluster_class(target: PosixPath, **kwargs): - """ - Update relevant files inside the cluster-stacks//cluster-class subdirectory. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - chart_file = target.joinpath("Chart.yaml") - values_file = target.joinpath("values.yaml") - - logger.info("Updating %s", chart_file) - content = readfile(chart_file) - version = get_dash_version(kwargs["kubernetes"]) - content["name"] = f"openstack-scs-{version}-cluster-class" - - writefile(chart_file, content) - - logger.info("Updating %s", values_file) - content = readfile(values_file) - - content["images"]["controlPlane"][ - "name" - ] = f"ubuntu-capi-image-v{kwargs['kubernetes']}" - content["images"]["worker"]["name"] = f"ubuntu-capi-image-v{kwargs['kubernetes']}" - - writefile(values_file, content) - - -def update_node_images(target: PosixPath, **kwargs): - """ - Update relevant files inside the cluster-stacks//node-images subdirectory. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - # TODO: can this magic URL be 'removed'? - # pylint: disable=locally-disabled, line-too-long - url = f"https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v{kwargs['kubernetes'][0:4]}/ubuntu-2204-kube-v{kwargs['kubernetes']}.qcow2" - content["spec"]["resource"]["content"]["download"]["url"] = url - - checksum_url = f"{url}.CHECKSUM" - response = requests.get(checksum_url, timeout=30) - response.raise_for_status() - - checksum_line = response.text.strip() - checksum = checksum_line.split()[0] - - content["spec"]["resource"]["content"]["download"]["hash"]["value"] = checksum - logger.info("Updated checksum: %s", checksum) - - writefile(target, content) - - -if __name__ == "__main__": - LOGFORMAT = "%(asctime)s - %(levelname)s: %(message)s" - logging.basicConfig(level=logging.INFO, encoding="utf-8", format=LOGFORMAT) - # Initialize arg parser - parser = argparse.ArgumentParser() - parser.add_argument( - "-t", - "--target-version", - type=str, - help="Generate files for version specified like 1.XX. See '-l' to list supported versions.", - ) - parser.add_argument( - "-l", "--list", action="store_true", help="List supported versions and exit." - ) - parser.add_argument("--build", action="store_true", help="Build helm dependencies.") - parser.add_argument( - "--build-verbose", action="store_false", help="Show output of helm build" - ) - args = parser.parse_args() - - # Load supported target versions - sup_versions = load_supported_versions() - - if args.list: - print("Supported Kubernetes Versions:") - for v in sup_versions: - print(f"{'.'.join(v['kubernetes'].split('.')[0:2])}") - print("Usage: generate_version.py --target-version VERSION") - sys.exit() - - # filter versions to generate - if args.target_version: - target_versions = [ - v for v in sup_versions if v["kubernetes"].startswith(args.target_version) - ] - else: - target_versions = sup_versions - - for tv in target_versions: - output_dir = create_output_dir(tv["kubernetes"]) - for chart_yaml in output_dir.joinpath("cluster-addon").rglob("Chart.yaml"): - update_cluster_addon( - chart_yaml, - args.build, - args.build_verbose, - **tv, - ) - update_csctl_conf(output_dir.joinpath("csctl.yaml"), **tv) - update_cluster_class(output_dir.joinpath("cluster-class"), **tv) - update_node_images( - output_dir.joinpath("cluster-class", "templates", "image.yaml"), **tv - ) diff --git a/hack/kind-dev.sh b/hack/kind-dev.sh deleted file mode 100755 index 70c2d99e..00000000 --- a/hack/kind-dev.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -x - -K8S_VERSION=v1.27.2 - -REPO_ROOT=$(git rev-parse --show-toplevel) -cd "${REPO_ROOT}" || exit 1 - -# Creates a kind cluster with the ctlptl tool https://github.com/tilt-dev/ctlptl -function ctlptl_kind-cluster() { - - local CLUSTER_NAME=$1 - local CLUSTER_VERSION=$2 - - cat < is omitted, it is derived from $PROVIDER and $CLUSTER_STACK +# (default: providers/openstack/scs). +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# OCI_REGISTRY OCI registry to query for CS versions (optional) +# OCI_REPOSITORY OCI repository to query for CS versions (optional) + +set -euo pipefail + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +require_command yq + +MARKDOWN=false +BASE_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --markdown) MARKDOWN=true; shift ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; exit 1 + fi + ;; + esac +done + +if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" +fi + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 +fi + +# ============================================ +# Collect data from all version directories +# ============================================ + +# First pass: discover all addon names across all versions +declare -A ALL_ADDON_NAMES +declare -A STACK_ADDONS # key: "1-XX:addon_name" → version + +PROVIDER_NAME="" +STACK_NAME="" + +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + local_dir=$(basename "$version_dir") + + stack_yaml="$version_dir/stack.yaml" + [[ -f "$stack_yaml" ]] || continue + + if [[ -z "$PROVIDER_NAME" ]]; then + PROVIDER_NAME=$(yq -r '.provider' "$stack_yaml") + STACK_NAME=$(yq -r '.clusterStackName' "$stack_yaml") + fi + + # Collect chart dependency versions + for chart_file in "$version_dir"/cluster-addon/*/Chart.yaml; do + [[ -f "$chart_file" ]] || continue + num_deps=$(yq '.dependencies | length' "$chart_file" 2>/dev/null || echo "0") + for ((i=0; i/dev/null 2>&1 && echo "true" || echo "false") + if [[ "$has_addons" == "true" ]]; then + addon_keys=$(yq -r '.addons | keys | .[]' "$stack_yaml") + for key in $addon_keys; do + value=$(yq -r ".addons.\"${key}\"" "$stack_yaml") + # Map short names to chart names for display + case "$key" in + ccm) chart_key="openstack-cloud-controller-manager" ;; + csi) chart_key="openstack-cinder-csi" ;; + *) chart_key="$key" ;; + esac + ALL_ADDON_NAMES["$chart_key"]=1 + # Mark with range (overrides chart default) + STACK_ADDONS["${local_dir}:${chart_key}"]="$value" + done + fi +done + +if [[ -z "$PROVIDER_NAME" ]]; then + echo "No valid version directories found in: $BASE_DIR" >&2 + exit 1 +fi + +# Sort addon names +SORTED_ADDONS=($(echo "${!ALL_ADDON_NAMES[@]}" | tr ' ' '\n' | sort)) + +# ============================================ +# Collect row data +# ============================================ + +declare -a ROW_VERSIONS=() +declare -a ROW_K8S=() +declare -a ROW_CS=() +declare -A ROW_ADDON_VERSIONS # key: "row_idx:addon_name" → version + +ROW_IDX=0 +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + local_dir=$(basename "$version_dir") + stack_yaml="$version_dir/stack.yaml" + [[ -f "$stack_yaml" ]] || continue + + k8s_version=$(yq -r '.kubernetesVersion' "$stack_yaml") + k8s_short=$(extract_k8s_minor_version "$k8s_version") + k8s_dash="${k8s_short//./-}" + + # Query OCI for CS version + CS_VERSION="-" + if [[ -n "${OCI_REGISTRY:-}" && -n "${OCI_REPOSITORY:-}" ]] && command -v oras >/dev/null 2>&1; then + TAG_PREFIX="${PROVIDER_NAME}-${STACK_NAME}-${k8s_dash}" + LATEST=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" 2>/dev/null | \ + extract_latest_release_number "$TAG_PREFIX" || echo "") + [[ -n "$LATEST" ]] && CS_VERSION="v${LATEST}" + fi + + ROW_VERSIONS+=("$local_dir") + ROW_K8S+=("$k8s_version") + ROW_CS+=("$CS_VERSION") + + for addon in "${SORTED_ADDONS[@]}"; do + ver="${STACK_ADDONS["${local_dir}:${addon}"]:-"-"}" + ROW_ADDON_VERSIONS["${ROW_IDX}:${addon}"]="$ver" + done + ((ROW_IDX++)) || true +done + +# Short display name for an addon +addon_short() { + echo "$1" | sed 's/openstack-cloud-controller-manager/os-ccm/; s/openstack-cinder-csi/os-csi/' +} + +# ============================================ +# Print the matrix +# ============================================ + +if [[ "$MARKDOWN" == true ]]; then + # --- Markdown table output --- + HEADER="| Version | K8s | CS Version" + SEPARATOR="|---------|-----|----------" + for addon in "${SORTED_ADDONS[@]}"; do + HEADER+=" | $(addon_short "$addon")" + SEPARATOR+="|---------" + done + HEADER+=" |" + SEPARATOR+="|" + echo "$HEADER" + echo "$SEPARATOR" + + for ((i=0; i is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). +# +# Options: +# --dry-run Preview changes without modifying files +# --all Run against all stacks in providers/*/*/ +# -h, --help Show this help +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# GITHUB_TOKEN Optional. GitHub personal access token for higher API rate limits. +# IMAGE_BASE_URL Base URL for CAPI images (default: https://nbg1.your-objectstorage.com/...) +# IMAGE_VISIBILITY Image visibility in image-manager.yaml (default: private) + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +readonly YQ_VERSION="$(yq --version 2>/dev/null || true)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_k8s_minor_number() { + echo "$1" | sed -E -n 's/^v?[0-9]+\.([0-9]+)(\.[0-9]+)?$/\1/p' +} + +yq_edit_in_place() { + local expression="$1" + local file="$2" + + if [[ "$YQ_VERSION" == *"https://github.com/mikefarah/yq/"* ]]; then + yq -i "$expression" "$file" + else + yq -y -i "$expression" "$file" + fi +} + +require_command yq + +require_versions_tools() { + require_command curl + require_command jq +} + +require_addons_tools() { + require_command helm + require_command jq +} + +# ============================================ +# Defaults +# ============================================ + +SUBCOMMAND="" # versions, addons, or "" (both) +BASE_DIR="" +DRY_RUN=false +RUN_ALL=false + +# Image-manager defaults (for OpenStack image manifest generation) +IMAGE_BASE_URL="${IMAGE_BASE_URL:-https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images}" +IMAGE_VISIBILITY="${IMAGE_VISIBILITY:-private}" + +# ============================================ +# Ubuntu version mapping +# ============================================ + +# K8s 1.32 and earlier → Ubuntu 22.04 (2204) +# K8s 1.33 and later → Ubuntu 24.04 (2404) +ubuntu_for_minor() { + local minor="$1" + if [[ "$minor" -le 32 ]]; then + echo "2204" + else + echo "2404" + fi +} + +# ============================================ +# Output helpers +# ============================================ + +info() { echo " $*"; } +ok() { echo " ✓ $*"; } +warn() { echo " ! $*"; } +change() { echo " → $*"; } + +# ============================================ +# Argument parsing +# ============================================ + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + versions|addons) + if [[ -n "$SUBCOMMAND" ]]; then + echo "Error: multiple subcommands given" >&2; usage + fi + SUBCOMMAND="$1"; shift + ;; + --dry-run) DRY_RUN=true; shift ;; + --all) RUN_ALL=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + # Resolve relative path + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +# Detect provider from stack dir path +detect_provider() { + local dir="$1" + local rel="${dir#"$REPO_ROOT"/}" + echo "$rel" | cut -d/ -f2 +} + +# ============================================ +# K8s version fetchers (provider-aware) +# ============================================ + +github_curl() { + local url="$1" + local -a headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + headers=(-H "Authorization: token $GITHUB_TOKEN") + fi + curl -sfL "${headers[@]+"${headers[@]}"}" "$url" +} + +fetch_k8s_versions_github() { + local page=1 + local all_tags="" + + while true; do + local tags + tags=$(github_curl "https://api.github.com/repos/kubernetes/kubernetes/tags?per_page=100&page=$page" \ + | jq -r '.[].name // empty') || { echo "Error fetching K8s tags from GitHub" >&2; return 1; } + + [[ -z "$tags" ]] && break + all_tags+="$tags"$'\n' + page=$((page + 1)) + + if echo "$tags" | grep -qE '^v1\.2[0-9]\.'; then + break + fi + done + + echo "$all_tags" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sed 's/^v//' | sort -V +} + +fetch_k8s_versions_dockerhub() { + local page=1 + local all_tags="" + + while true; do + local response + response=$(curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&page=$page") \ + || { echo "Error fetching kindest/node tags from Docker Hub" >&2; return 1; } + + local tags + tags=$(echo "$response" | jq -r '.results[].name // empty') + [[ -z "$tags" ]] && break + + all_tags+="$tags"$'\n' + + local next + next=$(echo "$response" | jq -r '.next // empty') + [[ -z "$next" ]] && break + + page=$((page + 1)) + + if echo "$tags" | grep -qE '^v1\.2[0-9]\.'; then + break + fi + done + + echo "$all_tags" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sed 's/^v//' | sort -V +} + +fetch_k8s_versions() { + local provider="$1" + case "$provider" in + docker) fetch_k8s_versions_dockerhub ;; + *) fetch_k8s_versions_github ;; + esac +} + +# Get the highest patch version for a given minor from a list of versions. +highest_patch() { + local minor="$1" + grep -E "^1\.${minor}\.[0-9]+$" | sort -V | tail -1 +} + +# ============================================ +# Helm repo URL maps for K8s-tied addons +# ============================================ + +declare -A ADDON_HELM_REPOS=( + ["openstack-cloud-controller-manager"]="https://kubernetes.github.io/cloud-provider-openstack" + ["openstack-cinder-csi"]="https://kubernetes.github.io/cloud-provider-openstack" +) + +declare -A ADDON_SHORT_TO_CHART=( + ["ccm"]="openstack-cloud-controller-manager" + ["csi"]="openstack-cinder-csi" +) + +# ============================================ +# Image-manager update (OpenStack only) +# ============================================ + +# Update providers///image-manager.yaml with merge semantics: +# - New minor versions are added +# - Existing minor versions are updated (new patch replaces old) +# - Minor versions no longer in 1-*/ dirs are preserved (never removed) +# +# Args: $1 = base_dir, $2 = name of associative array with resolved patches +update_image_manager() { + local base_dir="${1%/}" # strip trailing slash + local -n patches_ref=$2 # nameref to associative array + + local image_file="$base_dir/image-manager.yaml" + echo "Updating image-manager manifest: $image_file" + + # Build map of existing versions from file (if present), keyed by minor + # Each entry: "version|url|checksum" + declare -A existing_versions=() + if [[ -f "$image_file" ]]; then + local num_versions + num_versions=$(yq '.images[0].versions | length' "$image_file" 2>/dev/null || echo "0") + for ((i=0; i "$image_file" </dev/null || echo "0") + for ((i=0; i/dev/null 2>&1; then + echo "Failed to add Helm repo $dep_name ($dep_repo)." >&2 + exit 1 + fi + repos_added["$dep_name"]="$dep_repo" + fi + done + done + done + + if [[ ${#repos_added[@]} -gt 0 ]]; then + info "Updating Helm repos..." + if ! helm repo update > /dev/null 2>&1; then + echo "Failed to update Helm repositories." >&2 + exit 1 + fi + fi + echo "" + + local total_updates=0 + + for version_dir in "$base_dir"/1-*/; do + [[ -d "$version_dir" ]] || continue + local dir_name + dir_name=$(basename "$version_dir") + + echo "=== $dir_name ===" + + for addon_dir in "$version_dir"/cluster-addon/*/; do + [[ -f "$addon_dir/Chart.yaml" ]] || continue + local addon_name chart_file + addon_name=$(basename "$addon_dir") + chart_file="$addon_dir/Chart.yaml" + + local num_deps + num_deps=$(yq '.dependencies | length' "$chart_file" 2>/dev/null || echo "0") + [[ "$num_deps" == "0" || "$num_deps" == "null" ]] && continue + + for ((i=0; i/dev/null) + if [[ -n "$range" && "$range" != "null" ]]; then + is_tied=true + fi + fi + done + fi + + # Get latest version + local latest_version="" + if [[ "$is_tied" == "true" ]]; then + # For K8s-tied addons, match by prefix from stack.yaml range + local k8s_minor + k8s_minor=$(extract_k8s_minor_number "$(yq -r '.kubernetesVersion' "$stack_yaml")") + latest_version=$(helm search repo "$dep_name/$dep_name" --versions -o json 2>/dev/null | \ + jq -r --arg minor "$k8s_minor" \ + '[.[] | select(.version | startswith("2." + $minor + "."))] | .[0].version // empty' 2>/dev/null) || true + else + latest_version=$(helm search repo "$dep_name/$dep_name" -o json 2>/dev/null | \ + jq -r '.[0].version // empty' 2>/dev/null) || true + fi + + if [[ -z "$latest_version" ]]; then + warn "$dep_name: could not query upstream" + query_failures=$((query_failures + 1)) + continue + fi + + if [[ "$current_version" == "$latest_version" ]]; then + ok "$dep_name: $current_version (up to date)" + continue + fi + + change "$dep_name: $current_version → $latest_version" + total_updates=$((total_updates + 1)) + + if [[ "$DRY_RUN" != "true" ]]; then + yq_edit_in_place ".dependencies[$i].version = \"$latest_version\"" "$chart_file" + info "Updated $chart_file" + fi + done + done + echo "" + done + + if [[ "$query_failures" -gt 0 ]]; then + echo "$query_failures addon query failure(s) encountered." >&2 + exit 1 + fi + + if [[ "$DRY_RUN" == "true" ]]; then + if [[ "$total_updates" -gt 0 ]]; then + echo "$total_updates addon update(s) available (dry-run: no changes written)." + else + echo "All addons are up to date." + fi + elif [[ "$total_updates" -gt 0 ]]; then + echo "Applied $total_updates addon update(s)." + else + echo "All addons are up to date." + fi +} + +# ============================================ +# --all mode: iterate all stacks +# ============================================ + +run_all() { + local subcommand="$1" + local found=false + + # Find base dirs: providers/// that contain 1-*/ subdirs + for base_dir in "$REPO_ROOT"/providers/*/*/; do + # Must have at least one 1-*/ subdir + ls "$base_dir"/1-*/stack.yaml >/dev/null 2>&1 || continue + + found=true + echo "==========================================" + echo "$base_dir" + echo "==========================================" + + if [[ -z "$subcommand" ]]; then + cmd_versions "$base_dir" + echo "" + cmd_addons "$base_dir" + elif [[ "$subcommand" == "versions" ]]; then + cmd_versions "$base_dir" + elif [[ "$subcommand" == "addons" ]]; then + cmd_addons "$base_dir" + fi + + echo "" + done + + if [[ "$found" == false ]]; then + echo "No stacks found in providers/*/*/" >&2 + exit 1 + fi +} + +# ============================================ +# Main +# ============================================ + +main() { + parse_args "$@" + + if [[ -z "$SUBCOMMAND" || "$SUBCOMMAND" != "addons" ]]; then + require_versions_tools + fi + + if [[ -z "$SUBCOMMAND" || "$SUBCOMMAND" != "versions" ]]; then + require_addons_tools + fi + + if [[ "$RUN_ALL" == true ]]; then + run_all "$SUBCOMMAND" + return + fi + + BASE_DIR=$(resolve_base_dir "$BASE_DIR") + + if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 + fi + + if [[ -z "$SUBCOMMAND" ]]; then + cmd_versions "$BASE_DIR" + echo "" + cmd_addons "$BASE_DIR" + elif [[ "$SUBCOMMAND" == "versions" ]]; then + cmd_versions "$BASE_DIR" + elif [[ "$SUBCOMMAND" == "addons" ]]; then + cmd_addons "$BASE_DIR" + fi +} + +main "$@" diff --git a/justfile b/justfile new file mode 100644 index 00000000..e88644f3 --- /dev/null +++ b/justfile @@ -0,0 +1,181 @@ +# Cluster Stacks build system +# Usage: just [args...] +# Config: set PROVIDER and CLUSTER_STACK env vars or in .env (default: openstack/scs) +# +# All hack/ scripts derive the stack base directory from $PROVIDER and $CLUSTER_STACK +# automatically (e.g., providers/openstack/scs). Each base directory contains +# per-minor-version subdirs (1-32, 1-33, etc.) with self-contained stack.yaml. +# +# Container mode: set RUN_IN_CONTAINER=true to transparently run recipes inside +# the tools container. Build the image first with: just container-build + +set dotenv-load +set positional-arguments + +export PROVIDER := env("PROVIDER", "openstack") +export CLUSTER_STACK := env("CLUSTER_STACK", "scs") +RUN_IN_CONTAINER := env("RUN_IN_CONTAINER", "false") +CONTAINER_IMAGE := "cluster-stack-tools" + +# Show available recipes +default: + @just --list + +# ============================================ +# Build & Publish +# ============================================ + +# Build cluster-stack (e.g., just build --version 1.34 or just build --all) +build *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec build " + FLAGS } else { "./hack/build.sh " + FLAGS } }} + +# Build and publish (e.g., just publish --version 1.34 or just publish --all) +publish *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec publish " + FLAGS } else { "./hack/build.sh --publish " + FLAGS } }} + +# Build, publish, and generate the ClusterStack resource (e.g., just dev --version 1.35) +dev *FLAGS: + #!/usr/bin/env bash + set -euo pipefail + ./hack/build.sh --publish {{FLAGS}} + # Extract --version from flags for generate-resources + version="" + prev="" + for arg in {{FLAGS}}; do + if [[ "$prev" == "--version" ]]; then version="$arg"; fi + prev="$arg" + done + if [[ -n "$version" ]]; then + echo "" + echo "================================================================" + echo "ClusterStack resource (pipe to kubectl apply -f -)" + echo "================================================================" + ./hack/generate-resources.sh --version "$version" --clusterstack-only + fi + +# Install/upgrade the CSO with OCI config matching current environment +install-cso: + #!/usr/bin/env bash + set -euo pipefail + if [[ -z "${OCI_REGISTRY:-}" ]]; then + export OCI_REGISTRY="ttl.sh" + export OCI_REPOSITORY="clusterstacks-$(date +%Y%m%d)" + echo "Auto-configured ttl.sh: $OCI_REGISTRY/$OCI_REPOSITORY (expires in 24h)" + fi + CSO_CHART="${CSO_CHART:-oci://registry.scs.community/cluster-stacks/cso}" + echo "Installing/upgrading CSO..." + echo " Chart: $CSO_CHART" + echo " OCI config: $OCI_REGISTRY/${OCI_REPOSITORY:-}" + echo "" + helm upgrade -i cso "$CSO_CHART" \ + --namespace cso-system --create-namespace \ + --set controllerManager.manager.source=oci \ + --set "clusterStackVariables.ociRegistry=${OCI_REGISTRY}" \ + --set "clusterStackVariables.ociRepository=${OCI_REPOSITORY}" + +# Clean build artifacts +clean: + rm -rf .release + @echo "Cleaned .release" + +# ============================================ +# Update +# ============================================ + +# Update K8s versions and/or addon charts (subcommands: versions, addons) +update *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec update " + FLAGS } else { "./hack/update.sh " + FLAGS } }} + +# ============================================ +# Resource Generation +# ============================================ + +# Generate ClusterStack + Cluster YAML (e.g., just generate-resources --version 1.34) +generate-resources *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec generate-resources " + FLAGS } else { "./hack/generate-resources.sh " + FLAGS } }} + +# Generate only the ClusterStack resource +generate-clusterstack *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec generate-clusterstack " + FLAGS } else { "./hack/generate-resources.sh --clusterstack-only " + FLAGS } }} + +# Generate only the Cluster resource +generate-cluster *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec generate-cluster " + FLAGS } else { "./hack/generate-resources.sh --cluster-only " + FLAGS } }} + +# Generate OpenStack Image CRD manifests +generate-image-manifests *FLAGS: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec generate-image-manifests " + FLAGS } else { "./hack/generate-image-manifests.sh " + FLAGS } }} + +# ============================================ +# Info +# ============================================ + +# Show version matrix for all K8s versions and addons +matrix: + {{ if RUN_IN_CONTAINER == "true" { "just _container-exec matrix" } else { "./hack/show-matrix.sh" } }} + +# Generate configuration docs for all stacks +generate-docs: + #!/usr/bin/env bash + set -euo pipefail + template="hack/config-template.md" + # For each provider/stack, pick the latest version directory and generate docs + for stack_base in providers/*/*; do + [[ -d "$stack_base" ]] || continue + provider=$(basename "$(dirname "$stack_base")") + stack=$(basename "$stack_base") + # Find the latest version dir (highest 1-XX) + latest=$(ls -d "$stack_base"/1-*/ 2>/dev/null | sort -V | tail -1) + [[ -n "$latest" ]] || continue + outdir="docs/providers/${provider}" + outfile="${outdir}/${stack}-configuration.md" + mkdir -p "$outdir" + echo "Generating ${outfile} from ${latest} ..." + python3 ./hack/docugen.py "$latest" \ + --template "$template" \ + --matrix \ + --output "$outfile" + done + echo "Done." + +# ============================================ +# Container +# ============================================ + +# Detect container runtime (podman preferred, fallback to docker) +[private] +container-runtime: + #!/usr/bin/env bash + if command -v podman &>/dev/null; then echo "podman" + elif command -v docker &>/dev/null; then echo "docker" + else echo "ERROR: neither podman nor docker found" >&2; exit 1 + fi + +# Build the tools container image +container-build: + #!/usr/bin/env bash + set -euo pipefail + runtime=$(just container-runtime) + echo "Building {{CONTAINER_IMAGE}} with $runtime..." + $runtime build -t {{CONTAINER_IMAGE}} -f Containerfile . + +# Run a just recipe inside the container (used internally when RUN_IN_CONTAINER=true) +[private] +_container-exec +ARGS: + #!/usr/bin/env bash + set -euo pipefail + runtime=$(just container-runtime) + # Build image if it doesn't exist + if ! $runtime image exists {{CONTAINER_IMAGE}} 2>/dev/null && \ + ! $runtime images --format '{{"{{.Repository}}"}}' | grep -qx '{{CONTAINER_IMAGE}}'; then + echo "Image {{CONTAINER_IMAGE}} not found, building..." + just container-build + fi + $runtime run --rm -it \ + -v "$(pwd):/workspace" \ + -w /workspace \ + -e PROVIDER="$PROVIDER" \ + -e CLUSTER_STACK="$CLUSTER_STACK" \ + -e RUN_IN_CONTAINER=false \ + {{CONTAINER_IMAGE}} \ + just {{ARGS}} diff --git a/providers/docker/scs/cluster-addon/cni/Chart.yaml b/providers/docker/scs/1-32/cluster-addon/cni/Chart.yaml similarity index 89% rename from providers/docker/scs/cluster-addon/cni/Chart.yaml rename to providers/docker/scs/1-32/cluster-addon/cni/Chart.yaml index 69b6be9a..5b9b5dfa 100644 --- a/providers/docker/scs/cluster-addon/cni/Chart.yaml +++ b/providers/docker/scs/1-32/cluster-addon/cni/Chart.yaml @@ -7,4 +7,4 @@ dependencies: - alias: cilium name: cilium repository: https://helm.cilium.io/ - version: 1.16.6 + version: 1.19.1 diff --git a/providers/docker/scs/cluster-addon/cni/values.yaml b/providers/docker/scs/1-32/cluster-addon/cni/values.yaml similarity index 100% rename from providers/docker/scs/cluster-addon/cni/values.yaml rename to providers/docker/scs/1-32/cluster-addon/cni/values.yaml diff --git a/providers/docker/scs/cluster-addon/metrics-server/Chart.yaml b/providers/docker/scs/1-32/cluster-addon/metrics-server/Chart.yaml similarity index 91% rename from providers/docker/scs/cluster-addon/metrics-server/Chart.yaml rename to providers/docker/scs/1-32/cluster-addon/metrics-server/Chart.yaml index e5f961e3..a40982bf 100644 --- a/providers/docker/scs/cluster-addon/metrics-server/Chart.yaml +++ b/providers/docker/scs/1-32/cluster-addon/metrics-server/Chart.yaml @@ -5,6 +5,6 @@ name: yorizonpoc-metrics-server version: v1 dependencies: - name: "metrics-server" - version: "3.12.2" + version: "3.13.0" repository: "https://kubernetes-sigs.github.io/metrics-server/" alias: "metrics-server" diff --git a/providers/docker/scs/cluster-addon/metrics-server/values.yaml b/providers/docker/scs/1-32/cluster-addon/metrics-server/values.yaml similarity index 100% rename from providers/docker/scs/cluster-addon/metrics-server/values.yaml rename to providers/docker/scs/1-32/cluster-addon/metrics-server/values.yaml diff --git a/providers/docker/scs/cluster-class/Chart.yaml b/providers/docker/scs/1-32/cluster-class/Chart.yaml similarity index 67% rename from providers/docker/scs/cluster-class/Chart.yaml rename to providers/docker/scs/1-32/cluster-class/Chart.yaml index 3e2041e9..e24a03aa 100644 --- a/providers/docker/scs/cluster-class/Chart.yaml +++ b/providers/docker/scs/1-32/cluster-class/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 description: SCS Cluster Class -name: docker-scs-1-30-cluster-class +name: docker-scs-1-32-cluster-class type: application version: v1 diff --git a/providers/docker/scs/cluster-class/templates/_helpers.tpl b/providers/docker/scs/1-32/cluster-class/templates/_helpers.tpl similarity index 100% rename from providers/docker/scs/cluster-class/templates/_helpers.tpl rename to providers/docker/scs/1-32/cluster-class/templates/_helpers.tpl diff --git a/providers/docker/scs/cluster-class/templates/cluster-class.yaml b/providers/docker/scs/1-32/cluster-class/templates/cluster-class.yaml similarity index 100% rename from providers/docker/scs/cluster-class/templates/cluster-class.yaml rename to providers/docker/scs/1-32/cluster-class/templates/cluster-class.yaml diff --git a/providers/docker/scs/cluster-class/templates/docker-cluster-template.yaml b/providers/docker/scs/1-32/cluster-class/templates/docker-cluster-template.yaml similarity index 100% rename from providers/docker/scs/cluster-class/templates/docker-cluster-template.yaml rename to providers/docker/scs/1-32/cluster-class/templates/docker-cluster-template.yaml diff --git a/providers/docker/scs/cluster-class/templates/docker-machine-template.yaml b/providers/docker/scs/1-32/cluster-class/templates/docker-machine-template.yaml similarity index 100% rename from providers/docker/scs/cluster-class/templates/docker-machine-template.yaml rename to providers/docker/scs/1-32/cluster-class/templates/docker-machine-template.yaml diff --git a/providers/docker/scs/cluster-class/templates/kubeadm-config-template-worker-docker.yaml b/providers/docker/scs/1-32/cluster-class/templates/kubeadm-config-template-worker-docker.yaml similarity index 100% rename from providers/docker/scs/cluster-class/templates/kubeadm-config-template-worker-docker.yaml rename to providers/docker/scs/1-32/cluster-class/templates/kubeadm-config-template-worker-docker.yaml diff --git a/providers/docker/scs/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/docker/scs/1-32/cluster-class/templates/kubeadm-control-plane-template.yaml similarity index 100% rename from providers/docker/scs/cluster-class/templates/kubeadm-control-plane-template.yaml rename to providers/docker/scs/1-32/cluster-class/templates/kubeadm-control-plane-template.yaml diff --git a/providers/docker/scs/cluster-class/values.yaml b/providers/docker/scs/1-32/cluster-class/values.yaml similarity index 100% rename from providers/docker/scs/cluster-class/values.yaml rename to providers/docker/scs/1-32/cluster-class/values.yaml diff --git a/providers/docker/scs/clusteraddon.yaml b/providers/docker/scs/1-32/clusteraddon.yaml similarity index 100% rename from providers/docker/scs/clusteraddon.yaml rename to providers/docker/scs/1-32/clusteraddon.yaml diff --git a/providers/docker/scs/csctl.yaml b/providers/docker/scs/1-32/csctl.yaml similarity index 85% rename from providers/docker/scs/csctl.yaml rename to providers/docker/scs/1-32/csctl.yaml index c9d2c1cb..ccdc9cf1 100644 --- a/providers/docker/scs/csctl.yaml +++ b/providers/docker/scs/1-32/csctl.yaml @@ -1,7 +1,7 @@ apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 config: - kubernetesVersion: v1.30.10 clusterStackName: scs + kubernetesVersion: v1.32 provider: - type: docker apiVersion: docker.csctl.clusterstack.x-k8s.io/v1alpha1 + type: docker diff --git a/providers/docker/scs/1-32/stack.yaml b/providers/docker/scs/1-32/stack.yaml new file mode 100644 index 00000000..2efbee8f --- /dev/null +++ b/providers/docker/scs/1-32/stack.yaml @@ -0,0 +1,3 @@ +provider: docker +clusterStackName: scs +kubernetesVersion: 1.32 diff --git a/providers/docker/scs/1-33/cluster-addon/cni/Chart.yaml b/providers/docker/scs/1-33/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..7e37d894 --- /dev/null +++ b/providers/docker/scs/1-33/cluster-addon/cni/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: cni +description: Cilium CNI for Docker SCS2 cluster stack +type: application +version: 0.1.0 +dependencies: + - name: cilium + version: 1.19.1 + repository: https://helm.cilium.io/ diff --git a/providers/openstack/scs2/cluster-addon/cni/values.yaml b/providers/docker/scs/1-33/cluster-addon/cni/values.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/cni/values.yaml rename to providers/docker/scs/1-33/cluster-addon/cni/values.yaml diff --git a/providers/docker/scs/1-33/cluster-addon/metrics-server/Chart.yaml b/providers/docker/scs/1-33/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..764d3b7c --- /dev/null +++ b/providers/docker/scs/1-33/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: metrics-server +description: Metrics Server for Docker SCS2 cluster stack +type: application +version: 0.1.0 +dependencies: + - name: metrics-server + version: "3.13.0" + repository: https://kubernetes-sigs.github.io/metrics-server/ diff --git a/providers/openstack/scs/cluster-addon/metrics-server/overwrite.yaml b/providers/docker/scs/1-33/cluster-addon/metrics-server/overwrite.yaml similarity index 100% rename from providers/openstack/scs/cluster-addon/metrics-server/overwrite.yaml rename to providers/docker/scs/1-33/cluster-addon/metrics-server/overwrite.yaml diff --git a/providers/openstack/scs/cluster-addon/metrics-server/values.yaml b/providers/docker/scs/1-33/cluster-addon/metrics-server/values.yaml similarity index 100% rename from providers/openstack/scs/cluster-addon/metrics-server/values.yaml rename to providers/docker/scs/1-33/cluster-addon/metrics-server/values.yaml diff --git a/providers/docker/scs/1-33/cluster-class/Chart.yaml b/providers/docker/scs/1-33/cluster-class/Chart.yaml new file mode 100644 index 00000000..6471f5c0 --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: docker-scs-1-33-cluster-class +description: | + SCS Cluster Class (1.33) for the Docker infrastructure provider. + Uses CAPI v1beta1 APIs with camelCase variables (imageRepository, certSANs, oidcConfig). +type: application +version: v1 diff --git a/providers/docker/scs/1-33/cluster-class/templates/_helpers.tpl b/providers/docker/scs/1-33/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..62b7a97d --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/templates/_helpers.tpl @@ -0,0 +1,29 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/providers/docker/scs/1-33/cluster-class/templates/cluster-class.yaml b/providers/docker/scs/1-33/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..dbbbafaa --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,185 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} + namespace: {{ .Release.Namespace }} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} + machineInfrastructure: + ref: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + variables: + - name: imageRepository + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "registry.k8s.io" + description: "imageRepository sets the container registry to pull images from. If empty, the kubeadm default will be used." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. + usernameClaim: + type: string + example: "preferred_username" + default: "preferred_username" + description: "JWT claim to use as the user name." + groupsClaim: + type: string + example: "groups" + default: "groups" + description: "JWT claim to use as the user's group." + usernamePrefix: + type: string + example: "oidc:" + default: "oidc:" + description: "Prefix prepended to username claims to prevent clashes with existing names." + groupsPrefix: + type: string + example: "oidc:" + default: "oidc:" + description: "Prefix prepended to group claims to prevent clashes with existing names." + patches: + - name: imageRepository + description: "Sets the imageRepository used for the KubeadmControlPlane." + enabledIf: '{{ ne .imageRepository "" }}' + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository" + valueFrom: + variable: imageRepository + - name: customImageControlPlane + description: "Sets the container image for control plane DockerMachines." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.controlPlane 0).name }} + - name: customImageWorker + description: "Sets the container image for worker DockerMachines." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.worker 0).name }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-client-id" + valueFrom: + variable: oidcConfig.clientID + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-issuer-url" + valueFrom: + variable: oidcConfig.issuerURL + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-claim" + valueFrom: + variable: oidcConfig.usernameClaim + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-claim" + valueFrom: + variable: oidcConfig.groupsClaim + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-prefix" + valueFrom: + variable: oidcConfig.usernamePrefix + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-prefix" + valueFrom: + variable: oidcConfig.groupsPrefix diff --git a/providers/docker/scs/1-33/cluster-class/templates/docker-cluster-template.yaml b/providers/docker/scs/1-33/cluster-class/templates/docker-cluster-template.yaml new file mode 100644 index 00000000..0bdf7120 --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/templates/docker-cluster-template.yaml @@ -0,0 +1,8 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} +spec: + template: + spec: {} diff --git a/providers/docker/scs/1-33/cluster-class/templates/docker-machine-template.yaml b/providers/docker/scs/1-33/cluster-class/templates/docker-machine-template.yaml new file mode 100644 index 00000000..bc4c6cc6 --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/templates/docker-machine-template.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" diff --git a/providers/docker/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-docker.yaml b/providers/docker/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-docker.yaml new file mode 100644 index 00000000..ff13c65f --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-docker.yaml @@ -0,0 +1,14 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/docker/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml similarity index 71% rename from providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml rename to providers/docker/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml index 767eac68..5588759e 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/docker/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -2,21 +2,26 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} spec: template: spec: kubeadmConfigSpec: clusterConfiguration: - apiServer: {} + apiServer: + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] controllerManager: extraArgs: - cloud-provider: external + enable-hostpath-provisioner: "true" bind-address: 0.0.0.0 secure-port: "10257" + profiling: "false" + terminated-pod-gc-threshold: "100" scheduler: extraArgs: bind-address: 0.0.0.0 secure-port: "10259" + profiling: "false" etcd: local: dataDir: /var/lib/etcd @@ -36,12 +41,6 @@ spec: path: /etc/kube-proxy-config.yaml - content: | #!/usr/bin/env bash - - # - # (PK) I couldn't find a better/simpler way to conifgure it. See: - # https://github.com/kubernetes-sigs/cluster-api/issues/4512 - # - set -o errexit set -o nounset set -o pipefail @@ -49,23 +48,19 @@ spec: dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) readonly dir - # Exit fast if already appended. if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then exit 0 fi - # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). kubeadm_file="/etc/kubeadm.yml" if [[ ! -f ${kubeadm_file} ]]; then kubeadm_file="/run/kubeadm/kubeadm.yaml" fi - - # Run this script only if this is the init node. + if [[ ! -f ${kubeadm_file} ]]; then exit 0 fi - # Append kube-proxy-config.yaml to kubeadm config and delete it cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" rm "${dir}/kube-proxy-config.yaml" @@ -77,13 +72,13 @@ spec: - bash /etc/kube-proxy-patch.sh initConfiguration: nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock kubeletExtraArgs: - cloud-provider: external - provider-id: 'openstack:///{{ `{{ instance_id }}` }}' - name: '{{ `{{ local_hostname }}` }}' + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" joinConfiguration: nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock kubeletExtraArgs: - cloud-provider: external - provider-id: 'openstack:///{{ `{{ instance_id }}` }}' - name: '{{ `{{ local_hostname }}` }}' + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" diff --git a/providers/docker/scs/1-33/cluster-class/values.yaml b/providers/docker/scs/1-33/cluster-class/values.yaml new file mode 100644 index 00000000..29fe3394 --- /dev/null +++ b/providers/docker/scs/1-33/cluster-class/values.yaml @@ -0,0 +1,6 @@ +# Node images for kind (patched at build time by build.sh for each K8s version) +images: + controlPlane: + - name: registry.scs.community/docker.io/kindest/node:v1.34.3 + worker: + - name: registry.scs.community/docker.io/kindest/node:v1.34.3 diff --git a/providers/docker/scs/1-33/clusteraddon.yaml b/providers/docker/scs/1-33/clusteraddon.yaml new file mode 100644 index 00000000..9a639968 --- /dev/null +++ b/providers/docker/scs/1-33/clusteraddon.yaml @@ -0,0 +1,18 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply diff --git a/providers/docker/scs/1-33/csctl.yaml b/providers/docker/scs/1-33/csctl.yaml new file mode 100644 index 00000000..5c2283be --- /dev/null +++ b/providers/docker/scs/1-33/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.33 + provider: + apiVersion: docker.csctl.clusterstack.x-k8s.io/v1alpha1 + type: docker diff --git a/providers/docker/scs/1-33/stack.yaml b/providers/docker/scs/1-33/stack.yaml new file mode 100644 index 00000000..0a5fd917 --- /dev/null +++ b/providers/docker/scs/1-33/stack.yaml @@ -0,0 +1,3 @@ +provider: docker +clusterStackName: scs +kubernetesVersion: 1.33 diff --git a/providers/docker/scs/1-34/cluster-addon/cni/Chart.yaml b/providers/docker/scs/1-34/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..7e37d894 --- /dev/null +++ b/providers/docker/scs/1-34/cluster-addon/cni/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: cni +description: Cilium CNI for Docker SCS2 cluster stack +type: application +version: 0.1.0 +dependencies: + - name: cilium + version: 1.19.1 + repository: https://helm.cilium.io/ diff --git a/providers/docker/scs/1-34/cluster-addon/cni/values.yaml b/providers/docker/scs/1-34/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/docker/scs/1-34/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/docker/scs/1-34/cluster-addon/metrics-server/Chart.yaml b/providers/docker/scs/1-34/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..764d3b7c --- /dev/null +++ b/providers/docker/scs/1-34/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: metrics-server +description: Metrics Server for Docker SCS2 cluster stack +type: application +version: 0.1.0 +dependencies: + - name: metrics-server + version: "3.13.0" + repository: https://kubernetes-sigs.github.io/metrics-server/ diff --git a/providers/openstack/scs2/cluster-addon/metrics-server/overwrite.yaml b/providers/docker/scs/1-34/cluster-addon/metrics-server/overwrite.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/metrics-server/overwrite.yaml rename to providers/docker/scs/1-34/cluster-addon/metrics-server/overwrite.yaml diff --git a/providers/openstack/scs2/cluster-addon/metrics-server/values.yaml b/providers/docker/scs/1-34/cluster-addon/metrics-server/values.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/metrics-server/values.yaml rename to providers/docker/scs/1-34/cluster-addon/metrics-server/values.yaml diff --git a/providers/docker/scs/1-34/cluster-class/Chart.yaml b/providers/docker/scs/1-34/cluster-class/Chart.yaml new file mode 100644 index 00000000..1360581c --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: docker-scs-1-34-cluster-class +description: | + SCS Cluster Class (1.34) for the Docker infrastructure provider. + Uses CAPI v1beta1 APIs with camelCase variables (imageRepository, certSANs, oidcConfig). +type: application +version: v1 diff --git a/providers/docker/scs/1-34/cluster-class/templates/_helpers.tpl b/providers/docker/scs/1-34/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..62b7a97d --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/templates/_helpers.tpl @@ -0,0 +1,29 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/providers/docker/scs/1-34/cluster-class/templates/cluster-class.yaml b/providers/docker/scs/1-34/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..dbbbafaa --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,185 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} + namespace: {{ .Release.Namespace }} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} + machineInfrastructure: + ref: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + variables: + - name: imageRepository + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "registry.k8s.io" + description: "imageRepository sets the container registry to pull images from. If empty, the kubeadm default will be used." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. + usernameClaim: + type: string + example: "preferred_username" + default: "preferred_username" + description: "JWT claim to use as the user name." + groupsClaim: + type: string + example: "groups" + default: "groups" + description: "JWT claim to use as the user's group." + usernamePrefix: + type: string + example: "oidc:" + default: "oidc:" + description: "Prefix prepended to username claims to prevent clashes with existing names." + groupsPrefix: + type: string + example: "oidc:" + default: "oidc:" + description: "Prefix prepended to group claims to prevent clashes with existing names." + patches: + - name: imageRepository + description: "Sets the imageRepository used for the KubeadmControlPlane." + enabledIf: '{{ ne .imageRepository "" }}' + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository" + valueFrom: + variable: imageRepository + - name: customImageControlPlane + description: "Sets the container image for control plane DockerMachines." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.controlPlane 0).name }} + - name: customImageWorker + description: "Sets the container image for worker DockerMachines." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.worker 0).name }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-client-id" + valueFrom: + variable: oidcConfig.clientID + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-issuer-url" + valueFrom: + variable: oidcConfig.issuerURL + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-claim" + valueFrom: + variable: oidcConfig.usernameClaim + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-claim" + valueFrom: + variable: oidcConfig.groupsClaim + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-prefix" + valueFrom: + variable: oidcConfig.usernamePrefix + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-prefix" + valueFrom: + variable: oidcConfig.groupsPrefix diff --git a/providers/docker/scs/1-34/cluster-class/templates/docker-cluster-template.yaml b/providers/docker/scs/1-34/cluster-class/templates/docker-cluster-template.yaml new file mode 100644 index 00000000..0bdf7120 --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/templates/docker-cluster-template.yaml @@ -0,0 +1,8 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} +spec: + template: + spec: {} diff --git a/providers/docker/scs/1-34/cluster-class/templates/docker-machine-template.yaml b/providers/docker/scs/1-34/cluster-class/templates/docker-machine-template.yaml new file mode 100644 index 00000000..bc4c6cc6 --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/templates/docker-machine-template.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" diff --git a/providers/docker/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-docker.yaml b/providers/docker/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-docker.yaml new file mode 100644 index 00000000..ff13c65f --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-docker.yaml @@ -0,0 +1,14 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" diff --git a/providers/openstack/scs/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/docker/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml similarity index 70% rename from providers/openstack/scs/cluster-class/templates/kubeadm-control-plane-template.yaml rename to providers/docker/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml index 14e46f18..5588759e 100644 --- a/providers/openstack/scs/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/docker/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -2,23 +2,26 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} spec: template: spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - cloud-provider: external + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] controllerManager: extraArgs: - cloud-provider: external + enable-hostpath-provisioner: "true" bind-address: 0.0.0.0 secure-port: "10257" + profiling: "false" + terminated-pod-gc-threshold: "100" scheduler: extraArgs: bind-address: 0.0.0.0 secure-port: "10259" + profiling: "false" etcd: local: dataDir: /var/lib/etcd @@ -38,12 +41,6 @@ spec: path: /etc/kube-proxy-config.yaml - content: | #!/usr/bin/env bash - - # - # (PK) I couldn't find a better/simpler way to conifgure it. See: - # https://github.com/kubernetes-sigs/cluster-api/issues/4512 - # - set -o errexit set -o nounset set -o pipefail @@ -51,23 +48,19 @@ spec: dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) readonly dir - # Exit fast if already appended. if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then exit 0 fi - # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). kubeadm_file="/etc/kubeadm.yml" if [[ ! -f ${kubeadm_file} ]]; then kubeadm_file="/run/kubeadm/kubeadm.yaml" fi - - # Run this script only if this is the init node. + if [[ ! -f ${kubeadm_file} ]]; then exit 0 fi - # Append kube-proxy-config.yaml to kubeadm config and delete it cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" rm "${dir}/kube-proxy-config.yaml" @@ -79,13 +72,13 @@ spec: - bash /etc/kube-proxy-patch.sh initConfiguration: nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock kubeletExtraArgs: - cloud-provider: external - provider-id: openstack:///'{{"{{"}} instance_id {{"}}"}}' - name: '{{"{{"}} local_hostname {{"}}"}}' + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" joinConfiguration: nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock kubeletExtraArgs: - cloud-provider: external - provider-id: openstack:///'{{"{{"}} instance_id {{"}}"}}' - name: '{{"{{"}} local_hostname {{"}}"}}' + eviction-hard: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + fail-swap-on: "false" diff --git a/providers/docker/scs/1-34/cluster-class/values.yaml b/providers/docker/scs/1-34/cluster-class/values.yaml new file mode 100644 index 00000000..29fe3394 --- /dev/null +++ b/providers/docker/scs/1-34/cluster-class/values.yaml @@ -0,0 +1,6 @@ +# Node images for kind (patched at build time by build.sh for each K8s version) +images: + controlPlane: + - name: registry.scs.community/docker.io/kindest/node:v1.34.3 + worker: + - name: registry.scs.community/docker.io/kindest/node:v1.34.3 diff --git a/providers/docker/scs/1-34/clusteraddon.yaml b/providers/docker/scs/1-34/clusteraddon.yaml new file mode 100644 index 00000000..9a639968 --- /dev/null +++ b/providers/docker/scs/1-34/clusteraddon.yaml @@ -0,0 +1,18 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply diff --git a/providers/docker/scs/1-34/csctl.yaml b/providers/docker/scs/1-34/csctl.yaml new file mode 100644 index 00000000..83c6a077 --- /dev/null +++ b/providers/docker/scs/1-34/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.34 + provider: + apiVersion: docker.csctl.clusterstack.x-k8s.io/v1alpha1 + type: docker diff --git a/providers/docker/scs/1-34/stack.yaml b/providers/docker/scs/1-34/stack.yaml new file mode 100644 index 00000000..1e8da19d --- /dev/null +++ b/providers/docker/scs/1-34/stack.yaml @@ -0,0 +1,3 @@ +provider: docker +clusterStackName: scs +kubernetesVersion: 1.34 diff --git a/providers/docker/scs/1-35/cluster-addon/cni/Chart.yaml b/providers/docker/scs/1-35/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..1ee33dee --- /dev/null +++ b/providers/docker/scs/1-35/cluster-addon/cni/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: cni +description: Cilium CNI for Docker bryggan cluster stack +type: application +version: 0.1.0 +dependencies: + - name: cilium + version: 1.19.1 + repository: https://helm.cilium.io/ diff --git a/providers/docker/scs/1-35/cluster-addon/cni/values.yaml b/providers/docker/scs/1-35/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/docker/scs/1-35/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/docker/scs/1-35/cluster-addon/metrics-server/Chart.yaml b/providers/docker/scs/1-35/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..17cec027 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: metrics-server +description: Metrics Server for Docker bryggan cluster stack +type: application +version: 0.1.0 +dependencies: + - name: metrics-server + version: "3.13.0" + repository: https://kubernetes-sigs.github.io/metrics-server/ diff --git a/providers/docker/scs/cluster-addon-values.yaml b/providers/docker/scs/1-35/cluster-addon/metrics-server/overwrite.yaml similarity index 100% rename from providers/docker/scs/cluster-addon-values.yaml rename to providers/docker/scs/1-35/cluster-addon/metrics-server/overwrite.yaml diff --git a/providers/docker/scs/1-35/cluster-addon/metrics-server/values.yaml b/providers/docker/scs/1-35/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/docker/scs/1-35/cluster-class/Chart.yaml b/providers/docker/scs/1-35/cluster-class/Chart.yaml new file mode 100644 index 00000000..5d76f025 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: docker-scs-1-35-cluster-class +description: | + SCS Cluster Class (1.35) for the Docker infrastructure provider. + Uses CAPI v1beta2 for core/kubeadm resources, v1beta1 for Docker infra resources. +type: application +version: v1 diff --git a/providers/docker/scs/1-35/cluster-class/templates/_helpers.tpl b/providers/docker/scs/1-35/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..449a5720 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/providers/docker/scs/1-35/cluster-class/templates/cluster-class.yaml b/providers/docker/scs/1-35/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..a848c31a --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,314 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + machineInfrastructure: + templateRef: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + + # + # Variables + # + variables: + - name: imageRepository + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.imageRepository | quote }} + example: "registry.k8s.io" + description: "imageRepository sets the container registry to pull images from. If empty, the kubeadm default will be used." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.certSANs | toJson }} + example: ["mydomain.example"] + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to discover public signing keys. + Only URLs that use the https:// scheme are accepted. + usernameClaim: + type: string + example: "preferred_username" + default: {{ .Values.variables.oidcConfig.usernameClaim | quote }} + description: "JWT claim to use as the user name." + groupsClaim: + type: string + example: "groups" + default: {{ .Values.variables.oidcConfig.groupsClaim | quote }} + description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." + usernamePrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.usernamePrefix | quote }} + description: "Prefix prepended to username claims to prevent clashes with existing names." + groupsPrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.groupsPrefix | quote }} + description: "Prefix prepended to group claims to prevent clashes with existing names." + - name: registryMirrors + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.registryMirrors | toJson }} + description: "Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror." + items: + type: object + properties: + hostnameUpstream: + type: string + example: "docker.io" + description: "The hostname of the upstream registry." + urlUpstream: + type: string + example: "https://registry-1.docker.io" + description: "The server URL of the upstream registry." + urlMirror: + type: string + example: "https://registry.example.com/v2/dockerhub" + description: "The URL of the mirror registry." + certMirror: + type: string + example: "" + description: "TLS certificate of the mirror in PEM format (optional)." + + # + # Patches + # + patches: + - name: imageRepository + description: "Sets the imageRepository used for the KubeadmControlPlane." + enabledIf: {{ `'{{ ne .imageRepository "" }}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository" + valueFrom: + variable: imageRepository + - name: customImage + description: "Sets the container image that is used for running dockerMachines for the control plane." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.controlPlane 0).name }} + - name: workerImage + description: "Sets the container image that is used for running dockerMachines for the worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.worker 0).name }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-client-id + value: {{ `'{{ .oidcConfig.clientID }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-issuer-url + value: {{ `'{{ .oidcConfig.issuerURL }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-claim + value: {{ `'{{ .oidcConfig.usernameClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-claim + value: {{ `'{{ .oidcConfig.groupsClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-prefix + value: {{ `'{{ .oidcConfig.usernamePrefix }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-prefix + value: {{ `'{{ .oidcConfig.groupsPrefix }}'` }} + # + # Registry mirror patches + # + - name: registryMirrorsControlPlane + description: "Configure registry mirrors on control plane nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} + - name: registryMirrorsWorker + description: "Configure registry mirrors on worker nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} diff --git a/providers/docker/scs/1-35/cluster-class/templates/docker-cluster-template.yaml b/providers/docker/scs/1-35/cluster-class/templates/docker-cluster-template.yaml new file mode 100644 index 00000000..ca51270d --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/docker-cluster-template.yaml @@ -0,0 +1,7 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: {} diff --git a/providers/docker/scs/1-35/cluster-class/templates/docker-machine-template.yaml b/providers/docker/scs/1-35/cluster-class/templates/docker-machine-template.yaml new file mode 100644 index 00000000..372949f8 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/docker-machine-template.yaml @@ -0,0 +1,10 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" diff --git a/providers/docker/scs/1-35/cluster-class/templates/kubeadm-config-template-worker.yaml b/providers/docker/scs/1-35/cluster-class/templates/kubeadm-config-template-worker.yaml new file mode 100644 index 00000000..8117fedc --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/kubeadm-config-template-worker.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + - name: eviction-hard + value: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + - name: fail-swap-on + value: "false" diff --git a/providers/docker/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/docker/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..c4ee9070 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,99 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] + controllerManager: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10257" + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "100" + scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10259" + - name: profiling + value: "false" + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 + - name: auto-compaction-mode + value: periodic + - name: auto-compaction-retention + value: 8h + - name: election-timeout + value: "2500" + - name: heartbeat-interval + value: "250" + - name: snapshot-count + value: "6400" + files: + - content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + path: /etc/kube-proxy-config.yaml + - content: | + #!/usr/bin/env bash + set -o errexit + set -o nounset + set -o pipefail + + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + readonly dir + + if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then + exit 0 + fi + + kubeadm_file="/etc/kubeadm.yml" + if [[ ! -f ${kubeadm_file} ]]; then + kubeadm_file="/run/kubeadm/kubeadm.yaml" + fi + + if [[ ! -f ${kubeadm_file} ]]; then + exit 0 + fi + + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm "${dir}/kube-proxy-config.yaml" + + echo success > /tmp/kube-proxy-patch + owner: root:root + path: /etc/kube-proxy-patch.sh + permissions: "0755" + preKubeadmCommands: + - bash /etc/kube-proxy-patch.sh + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + - name: eviction-hard + value: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + - name: fail-swap-on + value: "false" + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + - name: eviction-hard + value: "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%" + - name: fail-swap-on + value: "false" diff --git a/providers/docker/scs/1-35/cluster-class/values.yaml b/providers/docker/scs/1-35/cluster-class/values.yaml new file mode 100644 index 00000000..bf6e6ea1 --- /dev/null +++ b/providers/docker/scs/1-35/cluster-class/values.yaml @@ -0,0 +1,20 @@ +# Node images for kind (patched at build time by build.sh for each K8s version) +images: + controlPlane: + - name: registry.scs.community/docker.io/kindest/node:v1.35.1 + worker: + - name: registry.scs.community/docker.io/kindest/node:v1.35.1 + +# ClusterClass variable defaults +# These are referenced by the ClusterClass template and can be overridden per deployment. +variables: + imageRepository: "" + certSANs: [] + registryMirrors: [] + oidcConfig: + clientID: "" + issuerURL: "" + usernameClaim: "preferred_username" + groupsClaim: "groups" + usernamePrefix: "oidc:" + groupsPrefix: "oidc:" diff --git a/providers/docker/scs/1-35/clusteraddon.yaml b/providers/docker/scs/1-35/clusteraddon.yaml new file mode 100644 index 00000000..9a639968 --- /dev/null +++ b/providers/docker/scs/1-35/clusteraddon.yaml @@ -0,0 +1,18 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply diff --git a/providers/docker/scs/1-35/csctl.yaml b/providers/docker/scs/1-35/csctl.yaml new file mode 100644 index 00000000..1d484941 --- /dev/null +++ b/providers/docker/scs/1-35/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.35 + provider: + apiVersion: docker.csctl.clusterstack.x-k8s.io/v1alpha1 + type: docker diff --git a/providers/docker/scs/1-35/stack.yaml b/providers/docker/scs/1-35/stack.yaml new file mode 100644 index 00000000..75f1e724 --- /dev/null +++ b/providers/docker/scs/1-35/stack.yaml @@ -0,0 +1,3 @@ +provider: docker +clusterStackName: scs +kubernetesVersion: 1.35 diff --git a/providers/docker/scs/README.md b/providers/docker/scs/README.md deleted file mode 100644 index 73e8ae8a..00000000 --- a/providers/docker/scs/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Cluster Stacks - -## Getting started - -```sh -# Create bootstrap cluster -echo " ---- -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: - ipFamily: dual -nodes: - - role: control-plane - extraMounts: - - hostPath: /var/run/docker.sock - containerPath: /var/run/docker.sock" | kind create cluster --config - - -# Init Cluster API -export CLUSTER_TOPOLOGY=true -export EXP_CLUSTER_RESOURCE_SET=true -export EXP_RUNTIME_SDK=true -clusterctl init --infrastructure docker - -kubectl -n capi-system rollout status deployment -kubectl -n capd-system rollout status deployment - -# Install CSO and CSPO -helm upgrade -i cso \ --n cso-system \ ---create-namespace \ -oci://registry.scs.community/cluster-stacks/cso \ ---set clusterStackVariables.ociRepository=registry.scs.community/kaas/cluster-stacks - -kubectl create namespace cluster -``` - -clusterstack.yaml - -```yaml -apiVersion: clusterstack.x-k8s.io/v1alpha1 -kind: ClusterStack -metadata: - name: docker - namespace: cluster -spec: - provider: docker - name: scs - kubernetesVersion: "1.30" - channel: custom - autoSubscribe: false - noProvider: true - versions: - - v0-sha.rwvgrna -``` - -Check if ClusterClasses exist - -```sh -kubectl get clusterclass -n cluster -``` - -cluster.yaml - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: docker-testcluster - namespace: cluster - labels: - managed-secret: cloud-config -spec: - topology: - class: docker-scs-1-30-v0-sha.rwvgrna - controlPlane: - replicas: 1 - version: v1.30.10 - workers: - machineDeployments: - - class: default-worker - name: md-0 - replicas: 1 -``` - -```sh -clusterctl get kubeconfig -n cluster docker-testcluster > /tmp/kubeconfig -kubectl get nodes --kubeconfig /tmp/kubeconfig -``` diff --git a/providers/docker/scs/cluster-addon/.helmignore b/providers/docker/scs/cluster-addon/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/docker/scs/cluster-addon/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/docker/scs/cluster-addon/cni/Chart.lock b/providers/docker/scs/cluster-addon/cni/Chart.lock deleted file mode 100644 index 776d3129..00000000 --- a/providers/docker/scs/cluster-addon/cni/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: cilium - repository: https://helm.cilium.io/ - version: 1.16.6 -digest: sha256:e6a746a27a71acab49c5d54cba2d37eed32e04f8b74af5651e2266ae251c55d8 -generated: "2025-02-13T12:55:17.200292016+01:00" diff --git a/providers/docker/scs/cluster-addon/cni/charts/cilium-1.16.6.tgz b/providers/docker/scs/cluster-addon/cni/charts/cilium-1.16.6.tgz deleted file mode 100644 index 3e99630f..00000000 Binary files a/providers/docker/scs/cluster-addon/cni/charts/cilium-1.16.6.tgz and /dev/null differ diff --git a/providers/docker/scs/cluster-addon/metrics-server/.helmignore b/providers/docker/scs/cluster-addon/metrics-server/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/docker/scs/cluster-addon/metrics-server/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/docker/scs/cluster-addon/metrics-server/Chart.lock b/providers/docker/scs/cluster-addon/metrics-server/Chart.lock deleted file mode 100644 index cb461d5a..00000000 --- a/providers/docker/scs/cluster-addon/metrics-server/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: metrics-server - repository: https://kubernetes-sigs.github.io/metrics-server/ - version: 3.12.2 -digest: sha256:b79715342d7c10e97664b5f4d79199044f5da6ef40cca906218cff05ca891122 -generated: "2025-01-13T15:40:51.780206883+01:00" diff --git a/providers/docker/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz b/providers/docker/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz deleted file mode 100644 index 4538e8a1..00000000 Binary files a/providers/docker/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz and /dev/null differ diff --git a/providers/docker/scs/cluster-class/.helmignore b/providers/docker/scs/cluster-class/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/docker/scs/cluster-class/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/docker/scs/clusterstack.yaml b/providers/docker/scs/clusterstack.yaml deleted file mode 100644 index 488aa714..00000000 --- a/providers/docker/scs/clusterstack.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: clusterstack.x-k8s.io/v1alpha1 -kind: ClusterStack -metadata: - name: docker-131 - namespace: cluster -spec: - autoSubscribe: false - channel: custom - kubernetesVersion: "1.31" - name: scs - noProvider: true - provider: docker - versions: - - v0-sha.hdl6pjy diff --git a/providers/openstack/scs2/cluster-addon/ccm/Chart.yaml b/providers/openstack/hcp/1-33/cluster-addon/ccm/Chart.yaml similarity index 92% rename from providers/openstack/scs2/cluster-addon/ccm/Chart.yaml rename to providers/openstack/hcp/1-33/cluster-addon/ccm/Chart.yaml index 7fddcbd4..541cecd0 100644 --- a/providers/openstack/scs2/cluster-addon/ccm/Chart.yaml +++ b/providers/openstack/hcp/1-33/cluster-addon/ccm/Chart.yaml @@ -7,4 +7,4 @@ dependencies: - alias: openstack-cloud-controller-manager name: openstack-cloud-controller-manager repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.34.1 + version: 2.33.1 diff --git a/providers/openstack/scs/cluster-addon/ccm/overwrite.yaml b/providers/openstack/hcp/1-33/cluster-addon/ccm/overwrite.yaml similarity index 100% rename from providers/openstack/scs/cluster-addon/ccm/overwrite.yaml rename to providers/openstack/hcp/1-33/cluster-addon/ccm/overwrite.yaml diff --git a/providers/openstack/scs2/cluster-addon/ccm/values.yaml b/providers/openstack/hcp/1-33/cluster-addon/ccm/values.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/ccm/values.yaml rename to providers/openstack/hcp/1-33/cluster-addon/ccm/values.yaml diff --git a/providers/openstack/scs2/cluster-addon/cni/Chart.yaml b/providers/openstack/hcp/1-33/cluster-addon/cni/Chart.yaml similarity index 88% rename from providers/openstack/scs2/cluster-addon/cni/Chart.yaml rename to providers/openstack/hcp/1-33/cluster-addon/cni/Chart.yaml index 7bc28cb7..051b40b3 100644 --- a/providers/openstack/scs2/cluster-addon/cni/Chart.yaml +++ b/providers/openstack/hcp/1-33/cluster-addon/cni/Chart.yaml @@ -7,4 +7,4 @@ dependencies: - alias: cilium name: cilium repository: https://helm.cilium.io/ - version: 1.18.5 + version: 1.19.1 diff --git a/providers/openstack/hcp/1-33/cluster-addon/cni/values.yaml b/providers/openstack/hcp/1-33/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/scs2/cluster-addon/csi/Chart.yaml b/providers/openstack/hcp/1-33/cluster-addon/csi/Chart.yaml similarity index 91% rename from providers/openstack/scs2/cluster-addon/csi/Chart.yaml rename to providers/openstack/hcp/1-33/cluster-addon/csi/Chart.yaml index dd303380..e275b2ff 100644 --- a/providers/openstack/scs2/cluster-addon/csi/Chart.yaml +++ b/providers/openstack/hcp/1-33/cluster-addon/csi/Chart.yaml @@ -7,4 +7,4 @@ dependencies: - alias: openstack-cinder-csi name: openstack-cinder-csi repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.34.1 + version: 2.33.1 diff --git a/providers/openstack/scs/cluster-addon/csi/overwrite.yaml b/providers/openstack/hcp/1-33/cluster-addon/csi/overwrite.yaml similarity index 100% rename from providers/openstack/scs/cluster-addon/csi/overwrite.yaml rename to providers/openstack/hcp/1-33/cluster-addon/csi/overwrite.yaml diff --git a/providers/openstack/scs2/cluster-addon/csi/values.yaml b/providers/openstack/hcp/1-33/cluster-addon/csi/values.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/csi/values.yaml rename to providers/openstack/hcp/1-33/cluster-addon/csi/values.yaml diff --git a/providers/openstack/scs2/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/hcp/1-33/cluster-addon/metrics-server/Chart.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/metrics-server/Chart.yaml rename to providers/openstack/hcp/1-33/cluster-addon/metrics-server/Chart.yaml diff --git a/providers/openstack/hcp/1-33/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/hcp/1-33/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/hcp/1-33/cluster-addon/metrics-server/values.yaml b/providers/openstack/hcp/1-33/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/hcp/1-33/cluster-class/Chart.yaml b/providers/openstack/hcp/1-33/cluster-class/Chart.yaml new file mode 100644 index 00000000..e6aa48aa --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-class/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +description: "OpenStack HCP (Hosted Control Plane) Cluster Class for K8s 1.33" +name: openstack-hcp-1-33-cluster-class +type: application +version: v1 diff --git a/providers/openstack/scs/cluster-class/templates/_helpers.tpl b/providers/openstack/hcp/1-33/cluster-class/templates/_helpers.tpl similarity index 100% rename from providers/openstack/scs/cluster-class/templates/_helpers.tpl rename to providers/openstack/hcp/1-33/cluster-class/templates/_helpers.tpl diff --git a/providers/openstack/scs2/cluster-class/templates/cluster-class.yaml b/providers/openstack/hcp/1-33/cluster-class/templates/cluster-class.yaml similarity index 67% rename from providers/openstack/scs2/cluster-class/templates/cluster-class.yaml rename to providers/openstack/hcp/1-33/cluster-class/templates/cluster-class.yaml index d7fbd338..dac6ba2c 100644 --- a/providers/openstack/scs2/cluster-class/templates/cluster-class.yaml +++ b/providers/openstack/hcp/1-33/cluster-class/templates/cluster-class.yaml @@ -5,14 +5,10 @@ metadata: spec: controlPlane: ref: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane - machineInfrastructure: - ref: - kind: OpenStackMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + # No machineInfrastructure section - control plane is hosted in management cluster infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -33,6 +29,29 @@ spec: kind: OpenStackMachineTemplate name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker variables: + # Control plane variables (hosted in management cluster) + - name: controlPlaneReplicas + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 1 + default: 3 + description: "Number of hosted control plane replicas for high availability." + - name: gatewayName + required: false + schema: + openAPIV3Schema: + type: string + default: "" + description: "Name of the Gateway API resource for control plane ingress. Optional - control plane accessible via standard kubeconfig if not set." + - name: gatewayNamespace + required: false + schema: + openAPIV3Schema: + type: string + default: "default" + description: "Namespace of the Gateway API resource." # Image variables - name: imageName required: false @@ -43,7 +62,7 @@ spec: The base name of the OpenStack image used for provisioning servers. If `imageIsOrc` is enabled, this name refers to an ORC image resource. If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. - If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5) + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.33.6) default: "ubuntu-capi-image" - name: imageIsOrc required: false @@ -61,16 +80,16 @@ spec: openAPIV3Schema: type: boolean description: | - Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5. + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.33.6. default: true - name: disableAPIServerFloatingIP required: false schema: openAPIV3Schema: type: boolean - default: false - example: false - description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + default: true + example: true + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server. Set to true for hosted control plane since API server runs in management cluster." # Network variables - name: networkExternalID required: false @@ -110,64 +129,15 @@ spec: Cluster actuator will create a network, a subnet with nodeCIDR, and a router connected to this subnet. If you leave this empty, no network will be created. - # Control plane - - name: controlPlaneFlavor - required: false - schema: - openAPIV3Schema: - type: string - default: "SCS-2V-4" - example: "SCS-2V-4-20s" - description: |- - OpenStack instance flavor for control plane nodes. - (Default: SCS-2V-4, replace by SCS-2V-4-20s or specify a controlPlaneRootDisk.) - - name: controlPlaneRootDisk - required: false - schema: - openAPIV3Schema: - type: integer - minimum: 0 - example: 25 - default: 50 - description: |- - Root disk size in GiB for control-plane nodes. - OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. - Should only be used for the diskless flavors (>= 20), otherwise set to 0. - - name: controlPlaneServerGroupID - required: false - schema: - openAPIV3Schema: - type: string - default: "" - example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" - description: "The server group to assign the control plane nodes to (can be used for anti-affinity)." - - name: controlPlaneAvailabilityZones - required: false - schema: - openAPIV3Schema: - type: array - example: ["nova"] - description: "controlPlaneAvailabilityZones is the set of availability zones which control plane machines may be deployed to." - items: - type: string - - name: controlPlaneOmitAvailabilityZone - required: false - schema: - openAPIV3Schema: - type: boolean - example: true - description: |- - controlPlaneOmitAvailabilityZone causes availability zone to be omitted when creating control plane nodes, - allowing the Nova scheduler to make a decision on which availability zone to use based on other scheduling constraints. # Workers - name: workerFlavor required: false schema: openAPIV3Schema: type: string - default: "SCS-4V-8" + default: {{ .Values.workerFlavor | quote }} example: "SCS-4V-8" - description: "OpenStack instance flavor for worker nodes (default: SCS-4V-8, which requires workerRootDisk)." + description: "OpenStack instance flavor for worker nodes." - name: workerRootDisk required: false schema: @@ -175,7 +145,7 @@ spec: type: integer minimum: 0 example: 25 - default: 50 + default: {{ .Values.workerRootDisk }} description: |- Root disk size in GiB for worker nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. @@ -223,7 +193,7 @@ spec: default: [] example: ["security-group-1"] description: |- - The names of extra security groups to assign to worker and control plane nodes. + The names of extra security groups to assign to worker nodes. Will be ignored if `securityGroupIDs` is used. items: type: string @@ -235,7 +205,7 @@ spec: type: array default: [] example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] - description: "The UUIDs of extra security groups to assign to worker and control plane nodes" + description: "The UUIDs of extra security groups to assign to worker nodes" items: type: string - name: workerSecurityGroups @@ -294,22 +264,20 @@ spec: schema: openAPIV3Schema: type: string - default: "octavia-ovn" + default: "none" example: "none, octavia-amphora, octavia-ovn" description: | - Cluster-API by default places a LoadBalancer in front of the kubernetes API server. - (There are also LBs that the CCM creates for a service type LoadBalancer which are configured independently.) - This setting here is to configure the LoadBalancer that is placed in front of the apiServer. + For hosted control plane, typically set to "none" since the API server runs in the management cluster. You can choose from 3 options: none: - No LoadBalancer solution will be deployed + (default) No LoadBalancer solution will be deployed octavia-amphora: Uses OpenStack's LoadBalancer service Octavia (provider:amphora) octavia-ovn: - (default) Uses OpenStack's LoadBalancer service Octavia (provider:ovn) + Uses OpenStack's LoadBalancer service Octavia (provider:ovn) - name: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs required: false schema: @@ -319,67 +287,61 @@ spec: description: |- apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs restrict access to the Kubernetes API server on a network level. Ensure that at least the outgoing IP of your Management Cluster is added to the list of allowed CIDRs. - Otherwise CAPO can’t reconcile the target Cluster correctly. + Otherwise CAPO can't reconcile the target Cluster correctly. This requires amphora as load balancer provider in version >= v2.12. items: type: string - - name: oidcConfig - required: false - schema: - openAPIV3Schema: - type: object - properties: - clientID: - type: string - example: "kubectl" - description: "A client id that all tokens must be issued for." - issuerURL: - type: string - example: "https://dex.k8s.scs.community" - description: >- - URL of the provider that allows the API server to - discover public signing keys. Only URLs that use the https:// scheme are - accepted. This is typically the provider's discovery URL, changed to have an - empty path. - usernameClaim: - type: string - example: "preferred_username" - default: "preferred_username" - description: >- - JWT claim to use as the user name. By default sub, - which is expected to be a unique identifier of the end user. Admins can choose - other claims, such as email or name, depending on their provider. However, - claims other than email will be prefixed with the issuer URL to prevent naming - clashes with other plugins. - groupsClaim: - type: string - example: "groups" - default: "groups" - description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." - usernamePrefix: - type: string - example: "oidc:" - default: "oidc:" - description: >- - Prefix prepended to username claims to prevent - clashes with existing names (such as system: users). For example, the value - oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and - --oidc-username-claim is a value other than email the prefix defaults to ( - Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value - - can be used to disable all prefixing. - groupsPrefix: - type: string - example: "oidc:" - default: "oidc:" - description: >- - Prefix prepended to group claims to prevent clashes - with existing names (such as system: groups). For example, the value oidc: will - create group names like oidc:engineering and oidc:infra. # # Patches # patches: # + # Patches for control plane variables (apply to HostedControlPlaneTemplate) + # + - name: controlPlaneReplicas + description: "Sets the number of hosted control plane replicas." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/replicas" + valueFrom: + variable: controlPlaneReplicas + - name: gatewayName + description: "Sets the Gateway API resource for control plane ingress." + enabledIf: {{ `'{{ ne .gatewayName "" }}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/gateway" + valueFrom: + template: | + name: {{ `{{ .gatewayName }}` }} + namespace: {{ `{{ .gatewayNamespace }}` }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServer/certSANs" + valueFrom: + variable: certSANs + # # Patches for OpenStackClusterTemplate resource. # - name: apiServerLoadBalancerOctaviaAmphora @@ -459,34 +421,6 @@ spec: path: "/spec/template/spec/networkMTU" valueFrom: variable: networkMTU - - name: controlPlaneAvailabilityZones - description: "Sets the availability zones which control plane machines may be deployed to." - enabledIf: {{ `'{{ if .controlPlaneAvailabilityZones }}true{{end}}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/controlPlaneAvailabilityZones" - valueFrom: - variable: controlPlaneAvailabilityZones - - name: controlPlaneOmitAvailabilityZone - description: "Causes availability zone to be omitted when creating control plane nodes." - enabledIf: {{ `'{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" - valueFrom: - variable: controlPlaneOmitAvailabilityZone - name: identityRef description: "Sets the OpenStack identity reference." definitions: @@ -537,128 +471,8 @@ spec: valueFrom: variable: disableAPIServerFloatingIP # - # Patches for control plane's OpenStackMachineTemplate resources. - # Note: Control plane patches are only applied when the control plane is managed by Kubeadm. - - name: controlPlaneImage - description: "Sets the OpenStack image name that is used for creating the control plane servers." - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: /spec/template/spec/image - valueFrom: - template: | - {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: - name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.controlPlane.version }}{{ end }}` }} - - name: controlPlaneFlavor - description: "Sets the openstack instance flavor for the KubeadmControlPlane." - enabledIf: {{ `'{{ ne .controlPlaneFlavor "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: replace - path: "/spec/template/spec/flavor" - valueFrom: - variable: controlPlaneFlavor - - name: controlPlaneRootDisk - description: "Sets the root disk size in GiB for control-plane nodes." - enabledIf: {{ `'{{ if .controlPlaneRootDisk }}true{{end}}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/rootVolume" - valueFrom: - template: | - sizeGiB: {{ `{{ .controlPlaneRootDisk }}` }} - - name: controlPlaneServerGroupID - description: "Sets the server group to assign the control plane nodes to." - enabledIf: {{ `'{{ ne .controlPlaneServerGroupID "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/serverGroup" - valueFrom: - template: | - id: {{ `{{ .controlPlaneServerGroupID }}` }} - # - # Patches for control plane's as well as worker's OpenStackMachineTemplate resources. - # Note: Control plane patches are only applied when the control plane is managed by Kubeadm. - # - # Note: The securityGroups patch must be placed before securityGroupIDs, workerSecurityGroups, and workerSecurityGroupIDs. - # The patch order ensures the last applied patch overwrites previous ones. - - name: securityGroups - description: "Sets the list of the openstack security groups for the worker and the control plane instances." - enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/securityGroups" - valueFrom: - template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} - # Note: The securityGroupIDs patch must be placed before workerSecurityGroups, workerSecurityGroupIDs and after securityGroupIDs. - # The patch order ensures the last applied patch overwrites previous ones. - - name: securityGroupIDs - description: "Sets the list of the openstack security groups for the worker and the control plane instances by UUID." - enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/securityGroups" - valueFrom: - template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} - - name: sshKeyName - description: "Sets the ssh key to inject in the nodes." - enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/sshKeyName" - valueFrom: - variable: sshKeyName - # # Patches for worker's OpenStackMachineTemplate resources. + # Note: No control plane patches since control plane runs in management cluster # - name: workerImage description: "Sets the OpenStack image name that is used for creating the worker servers." @@ -755,11 +569,11 @@ spec: volume: type: {{ `{{ .type }}` }} {{ `{{- end }}` }} - # Note: The workerSecurityGroups patch must be placed before workerSecurityGroupIDs and after securityGroups and securityGroupIDs. + # Note: The securityGroups patch must be placed before securityGroupIDs, workerSecurityGroups, and workerSecurityGroupIDs. # The patch order ensures the last applied patch overwrites previous ones. - - name: workerSecurityGroups + - name: securityGroups description: "Sets the list of the openstack security groups for the worker instances." - enabledIf: {{ `'{{ if .workerSecurityGroups }}true{{end}}'` }} + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} definitions: - selector: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -773,12 +587,12 @@ spec: - op: add path: "/spec/template/spec/securityGroups" valueFrom: - template: {{ `'[ {{ range .workerSecurityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} - # Note: The workerSecurityGroupIDs patch must be placed after securityGroups, securityGroupIDs and workerSecurityGroupIDs. + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The securityGroupIDs patch must be placed before workerSecurityGroups, workerSecurityGroupIDs and after securityGroupIDs. # The patch order ensures the last applied patch overwrites previous ones. - - name: workerSecurityGroupIDs + - name: securityGroupIDs description: "Sets the list of the openstack security groups for the worker instances by UUID." - enabledIf: {{ `'{{ if .workerSecurityGroupIDs }}true{{end}}'` }} + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} definitions: - selector: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -792,53 +606,59 @@ spec: - op: add path: "/spec/template/spec/securityGroups" valueFrom: - template: {{ `'[ {{ range .workerSecurityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} - # - - name: certSANs - description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." - enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the ssh key to inject in the nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} definitions: - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate matchResources: - controlPlane: true + controlPlane: false + machineDeploymentClass: + names: + - default-worker jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + path: "/spec/template/spec/sshKeyName" valueFrom: - variable: certSANs - - name: oidcConfig - description: "Configure API Server to use external authentication service." - enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + variable: sshKeyName + # Note: The workerSecurityGroups patch must be placed before workerSecurityGroupIDs and after securityGroups and securityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroups + description: "Sets the list of the openstack security groups for the worker instances." + enabledIf: {{ `'{{ if .workerSecurityGroups }}true{{end}}'` }} definitions: - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate matchResources: - controlPlane: true + controlPlane: false + machineDeploymentClass: + names: + - default-worker jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-client-id" - valueFrom: - variable: oidcConfig.clientID - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-issuer-url" - valueFrom: - variable: oidcConfig.issuerURL - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-claim" - valueFrom: - variable: oidcConfig.usernameClaim - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-claim" - valueFrom: - variable: oidcConfig.groupsClaim - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-prefix" + path: "/spec/template/spec/securityGroups" valueFrom: - variable: oidcConfig.usernamePrefix + template: {{ `'[ {{ range .workerSecurityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The workerSecurityGroupIDs patch must be placed after securityGroups, securityGroupIDs and workerSecurityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroupIDs + description: "Sets the list of the openstack security groups for the worker instances by UUID." + enabledIf: {{ `'{{ if .workerSecurityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-prefix" + path: "/spec/template/spec/securityGroups" valueFrom: - variable: oidcConfig.groupsPrefix + template: {{ `'[ {{ range .workerSecurityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} diff --git a/providers/openstack/hcp/1-33/cluster-class/templates/hosted-control-plane-template.yaml b/providers/openstack/hcp/1-33/cluster-class/templates/hosted-control-plane-template.yaml new file mode 100644 index 00000000..b1caeea8 --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-class/templates/hosted-control-plane-template.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: HostedControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + gateway: + namespace: overridden-by-patch + name: overridden-by-patch diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/hcp/1-33/cluster-class/templates/kubeadm-config-template-default-worker.yaml similarity index 100% rename from providers/openstack/scs2/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml rename to providers/openstack/hcp/1-33/cluster-class/templates/kubeadm-config-template-default-worker.yaml diff --git a/providers/openstack/scs2/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/hcp/1-33/cluster-class/templates/openstack-cluster-template.yaml similarity index 94% rename from providers/openstack/scs2/cluster-class/templates/openstack-cluster-template.yaml rename to providers/openstack/hcp/1-33/cluster-class/templates/openstack-cluster-template.yaml index 9d03326f..3b8ae609 100644 --- a/providers/openstack/scs2/cluster-class/templates/openstack-cluster-template.yaml +++ b/providers/openstack/hcp/1-33/cluster-class/templates/openstack-cluster-template.yaml @@ -11,10 +11,10 @@ spec: name: overridden-by-patch apiServerLoadBalancer: enabled: false + disableAPIServerFloatingIP: true managedSecurityGroups: allNodesSecurityGroupRules: - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 @@ -24,7 +24,6 @@ spec: protocol: udp description: "Allow VXLAN traffic for Cilium" - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 @@ -34,7 +33,6 @@ spec: protocol: tcp description: "Allow HealthCheck traffic for Cilium" - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 diff --git a/providers/openstack/scs2/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/hcp/1-33/cluster-class/templates/openstack-machine-template-default-worker.yaml similarity index 100% rename from providers/openstack/scs2/cluster-class/templates/openstack-machine-template-worker.yaml rename to providers/openstack/hcp/1-33/cluster-class/templates/openstack-machine-template-default-worker.yaml diff --git a/providers/openstack/hcp/1-33/cluster-class/values.yaml b/providers/openstack/hcp/1-33/cluster-class/values.yaml new file mode 100644 index 00000000..5ebb9d61 --- /dev/null +++ b/providers/openstack/hcp/1-33/cluster-class/values.yaml @@ -0,0 +1,24 @@ +# Default values for hosted-control-plane cluster class + +# OpenStack credentials +identityRef: + name: "openstack" + cloudName: "openstack" + +# Network configuration +networkExternalID: "" +dnsNameservers: + - "9.9.9.9" + - "149.112.112.112" +nodeCIDR: "10.8.0.0/20" + +# Worker node configuration +workerFlavor: "SCS-2V-4" +workerRootDisk: 25 + +# Control plane configuration (hosted in management cluster) +controlPlaneReplicas: 3 + +# Gateway configuration for hosted control plane (optional) +gatewayName: "" +gatewayNamespace: "" diff --git a/providers/openstack/scs/clusteraddon.yaml b/providers/openstack/hcp/1-33/clusteraddon.yaml similarity index 100% rename from providers/openstack/scs/clusteraddon.yaml rename to providers/openstack/hcp/1-33/clusteraddon.yaml diff --git a/providers/openstack/scs2/csctl.yaml b/providers/openstack/hcp/1-33/csctl.yaml similarity index 73% rename from providers/openstack/scs2/csctl.yaml rename to providers/openstack/hcp/1-33/csctl.yaml index d56bbbe7..6cb8a11b 100644 --- a/providers/openstack/scs2/csctl.yaml +++ b/providers/openstack/hcp/1-33/csctl.yaml @@ -1,7 +1,7 @@ apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 config: - clusterStackName: scs2 - kubernetesVersion: v1.34.3 + clusterStackName: hcp + kubernetesVersion: v1.33 provider: apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 type: openstack diff --git a/providers/openstack/hcp/1-33/stack.yaml b/providers/openstack/hcp/1-33/stack.yaml new file mode 100644 index 00000000..43c10ac5 --- /dev/null +++ b/providers/openstack/hcp/1-33/stack.yaml @@ -0,0 +1,7 @@ +provider: openstack +clusterStackName: hcp +kubernetesVersion: 1.33 + +addons: + ccm: 2.33.x + csi: 2.33.x diff --git a/providers/openstack/hcp/1-34/cluster-addon/ccm/Chart.yaml b/providers/openstack/hcp/1-34/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..bd086402 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.34.2 diff --git a/providers/openstack/scs2/cluster-addon/ccm/overwrite.yaml b/providers/openstack/hcp/1-34/cluster-addon/ccm/overwrite.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/ccm/overwrite.yaml rename to providers/openstack/hcp/1-34/cluster-addon/ccm/overwrite.yaml diff --git a/providers/openstack/hcp/1-34/cluster-addon/ccm/values.yaml b/providers/openstack/hcp/1-34/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/hcp/1-34/cluster-addon/cni/Chart.yaml b/providers/openstack/hcp/1-34/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..051b40b3 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.1 diff --git a/providers/openstack/hcp/1-34/cluster-addon/cni/values.yaml b/providers/openstack/hcp/1-34/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/hcp/1-34/cluster-addon/csi/Chart.yaml b/providers/openstack/hcp/1-34/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..cbe0cc17 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.34.3 diff --git a/providers/openstack/scs2/cluster-addon/csi/overwrite.yaml b/providers/openstack/hcp/1-34/cluster-addon/csi/overwrite.yaml similarity index 100% rename from providers/openstack/scs2/cluster-addon/csi/overwrite.yaml rename to providers/openstack/hcp/1-34/cluster-addon/csi/overwrite.yaml diff --git a/providers/openstack/hcp/1-34/cluster-addon/csi/values.yaml b/providers/openstack/hcp/1-34/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/hcp/1-34/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/hcp/1-34/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/hcp/1-34/cluster-addon/metrics-server/values.yaml b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/hcp/1-34/cluster-class/Chart.yaml b/providers/openstack/hcp/1-34/cluster-class/Chart.yaml new file mode 100644 index 00000000..4874c66d --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-class/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +description: "OpenStack HCP (Hosted Control Plane) Cluster Class for K8s 1.34" +name: openstack-hcp-1-34-cluster-class +type: application +version: v1 diff --git a/providers/openstack/scs2/cluster-class/templates/_helpers.tpl b/providers/openstack/hcp/1-34/cluster-class/templates/_helpers.tpl similarity index 100% rename from providers/openstack/scs2/cluster-class/templates/_helpers.tpl rename to providers/openstack/hcp/1-34/cluster-class/templates/_helpers.tpl diff --git a/providers/openstack/hcp/1-34/cluster-class/templates/cluster-class.yaml b/providers/openstack/hcp/1-34/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..dac6ba2c --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,664 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + # No machineInfrastructure section - control plane is hosted in management cluster + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Control plane variables (hosted in management cluster) + - name: controlPlaneReplicas + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 1 + default: 3 + description: "Number of hosted control plane replicas for high availability." + - name: gatewayName + required: false + schema: + openAPIV3Schema: + type: string + default: "" + description: "Name of the Gateway API resource for control plane ingress. Optional - control plane accessible via standard kubeconfig if not set." + - name: gatewayNamespace + required: false + schema: + openAPIV3Schema: + type: string + default: "default" + description: "Namespace of the Gateway API resource." + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.33.6) + default: "ubuntu-capi-image" + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: false + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.33.6. + default: true + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: true + example: true + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server. Set to true for hosted control plane since API server runs in management cluster." + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: "networkExternalID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs in case there are several external networks." + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "networkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "dnsNameservers is the list of nameservers for the OpenStack Subnet being created. Set this value when you need to create a new network/subnet which requires access to DNS." + default: ["9.9.9.9", "149.112.112.112"] + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: "10.8.0.0/20" + example: "10.8.0.0/20" + description: |- + nodeCIDR is the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with nodeCIDR, + and a router connected to this subnet. + If you leave this empty, no network will be created. + # Workers + - name: workerFlavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.workerFlavor | quote }} + example: "SCS-4V-8" + description: "OpenStack instance flavor for worker nodes." + - name: workerRootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 25 + default: {{ .Values.workerRootDisk }} + description: |- + Root disk size in GiB for worker nodes. + OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. + Should be used for the diskless flavors (>= 20), otherwise set to 0. + - name: workerServerGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "869fe071-1e56-46a9-9166-47c9f228e297" + description: "The server group to assign the worker nodes to." + - name: workerAdditionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: [] + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "capi-keypair" + description: "The ssh key name to inject in the nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["security-group-1"] + description: |- + The names of extra security groups to assign to worker nodes. + Will be ignored if `securityGroupIDs` is used. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: [] + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: "The UUIDs of extra security groups to assign to worker nodes" + items: + type: string + - name: workerSecurityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["security-group-1"] + description: |- + The names of extra security groups to assign to the worker nodes. + Will be ignored if `workerSecurityGroupIDs` is used. + items: + type: string + - name: workerSecurityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: [] + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: "The UUIDs of extra security groups to assign to the worker nodes" + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {} + properties: + name: + type: string + example: "openstack" + default: "openstack" + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: "openstack" + description: "The name of the cloud to use from the clouds.yaml" + # Kubernetes API server + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["mydomain.example"] + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: "none" + example: "none, octavia-amphora, octavia-ovn" + description: | + For hosted control plane, typically set to "none" since the API server runs in the management cluster. + You can choose from 3 options: + + none: + (default) No LoadBalancer solution will be deployed + + octavia-amphora: + Uses OpenStack's LoadBalancer service Octavia (provider:amphora) + + octavia-ovn: + Uses OpenStack's LoadBalancer service Octavia (provider:ovn) + - name: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs restrict access to the Kubernetes API server on a network level. + Ensure that at least the outgoing IP of your Management Cluster is added to the list of allowed CIDRs. + Otherwise CAPO can't reconcile the target Cluster correctly. + This requires amphora as load balancer provider in version >= v2.12. + items: + type: string + # + # Patches + # + patches: + # + # Patches for control plane variables (apply to HostedControlPlaneTemplate) + # + - name: controlPlaneReplicas + description: "Sets the number of hosted control plane replicas." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/replicas" + valueFrom: + variable: controlPlaneReplicas + - name: gatewayName + description: "Sets the Gateway API resource for control plane ingress." + enabledIf: {{ `'{{ ne .gatewayName "" }}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/gateway" + valueFrom: + template: | + name: {{ `{{ .gatewayName }}` }} + namespace: {{ `{{ .gatewayNamespace }}` }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServer/certSANs" + valueFrom: + variable: certSANs + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + description: "Takes care of the patches that should be applied when variable allowedCIDRs is set." + enabledIf: {{ `'{{ and .apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + # + # Patches for worker's OpenStackMachineTemplate resources. + # Note: No control plane patches since control plane runs in management cluster + # + - name: workerImage + description: "Sets the OpenStack image name that is used for creating the worker servers." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + - name: workerFlavor + description: "Sets the openstack instance flavor for the worker nodes." + enabledIf: {{ `'{{ ne .workerFlavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: workerFlavor + - name: workerRootDisk + description: "Sets the root disk size in GiB for worker nodes." + enabledIf: {{ `'{{ if .workerRootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .workerRootDisk }}` }} + - name: workerServerGroupID + description: "Sets the server group to assign the worker nodes to." + enabledIf: {{ `'{{ ne .workerServerGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .workerServerGroupID }}` }} + - name: workerAdditionalBlockDevices + enabledIf: {{ `'{{ if .workerAdditionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .workerAdditionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + # Note: The securityGroups patch must be placed before securityGroupIDs, workerSecurityGroups, and workerSecurityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: securityGroups + description: "Sets the list of the openstack security groups for the worker instances." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The securityGroupIDs patch must be placed before workerSecurityGroups, workerSecurityGroupIDs and after securityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: securityGroupIDs + description: "Sets the list of the openstack security groups for the worker instances by UUID." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the ssh key to inject in the nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # Note: The workerSecurityGroups patch must be placed before workerSecurityGroupIDs and after securityGroups and securityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroups + description: "Sets the list of the openstack security groups for the worker instances." + enabledIf: {{ `'{{ if .workerSecurityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .workerSecurityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The workerSecurityGroupIDs patch must be placed after securityGroups, securityGroupIDs and workerSecurityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroupIDs + description: "Sets the list of the openstack security groups for the worker instances by UUID." + enabledIf: {{ `'{{ if .workerSecurityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .workerSecurityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} diff --git a/providers/openstack/hcp/1-34/cluster-class/templates/hosted-control-plane-template.yaml b/providers/openstack/hcp/1-34/cluster-class/templates/hosted-control-plane-template.yaml new file mode 100644 index 00000000..b1caeea8 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-class/templates/hosted-control-plane-template.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: HostedControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + gateway: + namespace: overridden-by-patch + name: overridden-by-patch diff --git a/providers/openstack/scs/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/hcp/1-34/cluster-class/templates/kubeadm-config-template-default-worker.yaml similarity index 70% rename from providers/openstack/scs/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml rename to providers/openstack/hcp/1-34/cluster-class/templates/kubeadm-config-template-default-worker.yaml index 2e9ff290..4c1494ed 100644 --- a/providers/openstack/scs/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml +++ b/providers/openstack/hcp/1-34/cluster-class/templates/kubeadm-config-template-default-worker.yaml @@ -9,5 +9,5 @@ spec: nodeRegistration: kubeletExtraArgs: cloud-provider: external - provider-id: openstack:///'{{"{{"}} instance_id {{"}}"}}' - name: '{{"{{"}} local_hostname {{"}}"}}' + provider-id: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/hcp/1-34/cluster-class/templates/openstack-cluster-template.yaml similarity index 76% rename from providers/openstack/scs/cluster-class/templates/openstack-cluster-template.yaml rename to providers/openstack/hcp/1-34/cluster-class/templates/openstack-cluster-template.yaml index ce689a2a..3b8ae609 100644 --- a/providers/openstack/scs/cluster-class/templates/openstack-cluster-template.yaml +++ b/providers/openstack/hcp/1-34/cluster-class/templates/openstack-cluster-template.yaml @@ -1,3 +1,4 @@ +--- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: OpenStackClusterTemplate metadata: @@ -6,17 +7,14 @@ spec: template: spec: identityRef: - cloudName: {{ .Values.identityRef.cloudName }} - name: {{ .Values.identityRef.name }} + cloudName: overridden-by-patch + name: overridden-by-patch apiServerLoadBalancer: - enabled: {{ .Values.openstack_loadbalancer_apiserver }} -{{- if .Values.restrict_kubeapi }} - allowedCIDRs: {{ .Values.restrict_kubeapi }} -{{- end }} + enabled: false + disableAPIServerFloatingIP: true managedSecurityGroups: allNodesSecurityGroupRules: - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 @@ -26,7 +24,6 @@ spec: protocol: udp description: "Allow VXLAN traffic for Cilium" - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 @@ -36,7 +33,6 @@ spec: protocol: tcp description: "Allow HealthCheck traffic for Cilium" - remoteManagedGroups: - - controlplane - worker direction: ingress etherType: IPv4 diff --git a/providers/openstack/hcp/1-34/cluster-class/templates/openstack-machine-template-default-worker.yaml b/providers/openstack/hcp/1-34/cluster-class/templates/openstack-machine-template-default-worker.yaml new file mode 100644 index 00000000..920dbb0a --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-class/templates/openstack-machine-template-default-worker.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/hcp/1-34/cluster-class/values.yaml b/providers/openstack/hcp/1-34/cluster-class/values.yaml new file mode 100644 index 00000000..5ebb9d61 --- /dev/null +++ b/providers/openstack/hcp/1-34/cluster-class/values.yaml @@ -0,0 +1,24 @@ +# Default values for hosted-control-plane cluster class + +# OpenStack credentials +identityRef: + name: "openstack" + cloudName: "openstack" + +# Network configuration +networkExternalID: "" +dnsNameservers: + - "9.9.9.9" + - "149.112.112.112" +nodeCIDR: "10.8.0.0/20" + +# Worker node configuration +workerFlavor: "SCS-2V-4" +workerRootDisk: 25 + +# Control plane configuration (hosted in management cluster) +controlPlaneReplicas: 3 + +# Gateway configuration for hosted control plane (optional) +gatewayName: "" +gatewayNamespace: "" diff --git a/providers/openstack/scs2/clusteraddon.yaml b/providers/openstack/hcp/1-34/clusteraddon.yaml similarity index 100% rename from providers/openstack/scs2/clusteraddon.yaml rename to providers/openstack/hcp/1-34/clusteraddon.yaml diff --git a/providers/openstack/hcp/1-34/csctl.yaml b/providers/openstack/hcp/1-34/csctl.yaml new file mode 100644 index 00000000..81f85d82 --- /dev/null +++ b/providers/openstack/hcp/1-34/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: hcp + kubernetesVersion: v1.34 + provider: + apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 + type: openstack diff --git a/providers/openstack/hcp/1-34/stack.yaml b/providers/openstack/hcp/1-34/stack.yaml new file mode 100644 index 00000000..377cc46a --- /dev/null +++ b/providers/openstack/hcp/1-34/stack.yaml @@ -0,0 +1,7 @@ +provider: openstack +clusterStackName: hcp +kubernetesVersion: 1.34 + +addons: + ccm: 2.34.x + csi: 2.34.x diff --git a/providers/openstack/hcp/1-35/cluster-addon/ccm/Chart.yaml b/providers/openstack/hcp/1-35/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..1a120416 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.35.0 diff --git a/providers/openstack/hcp/1-35/cluster-addon/ccm/overwrite.yaml b/providers/openstack/hcp/1-35/cluster-addon/ccm/overwrite.yaml new file mode 100644 index 00000000..39076ecd --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/ccm/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + openstack-cloud-controller-manager: + cluster: + name: {{ .Cluster.metadata.name }} diff --git a/providers/openstack/hcp/1-35/cluster-addon/ccm/values.yaml b/providers/openstack/hcp/1-35/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/hcp/1-35/cluster-addon/cni/Chart.yaml b/providers/openstack/hcp/1-35/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..051b40b3 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.1 diff --git a/providers/openstack/hcp/1-35/cluster-addon/cni/values.yaml b/providers/openstack/hcp/1-35/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/hcp/1-35/cluster-addon/csi/Chart.yaml b/providers/openstack/hcp/1-35/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..b7d9ee53 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.35.0 diff --git a/providers/openstack/hcp/1-35/cluster-addon/csi/overwrite.yaml b/providers/openstack/hcp/1-35/cluster-addon/csi/overwrite.yaml new file mode 100644 index 00000000..d191a115 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/csi/overwrite.yaml @@ -0,0 +1,3 @@ +values: | + openstack-cinder-csi: + clusterID: "{{ .Cluster.metadata.name }}" diff --git a/providers/openstack/hcp/1-35/cluster-addon/csi/values.yaml b/providers/openstack/hcp/1-35/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/hcp/1-35/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/hcp/1-35/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/hcp/1-35/cluster-addon/metrics-server/values.yaml b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/hcp/1-35/cluster-class/Chart.yaml b/providers/openstack/hcp/1-35/cluster-class/Chart.yaml new file mode 100644 index 00000000..13d1cadb --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +description: "OpenStack HCP (Hosted Control Plane) Cluster Class for K8s 1.35" +name: openstack-hcp-1-35-cluster-class +type: application +version: v1 diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/_helpers.tpl b/providers/openstack/hcp/1-35/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/cluster-class.yaml b/providers/openstack/hcp/1-35/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..dac6ba2c --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,664 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + # No machineInfrastructure section - control plane is hosted in management cluster + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Control plane variables (hosted in management cluster) + - name: controlPlaneReplicas + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 1 + default: 3 + description: "Number of hosted control plane replicas for high availability." + - name: gatewayName + required: false + schema: + openAPIV3Schema: + type: string + default: "" + description: "Name of the Gateway API resource for control plane ingress. Optional - control plane accessible via standard kubeconfig if not set." + - name: gatewayNamespace + required: false + schema: + openAPIV3Schema: + type: string + default: "default" + description: "Namespace of the Gateway API resource." + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.33.6) + default: "ubuntu-capi-image" + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: false + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.33.6. + default: true + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: true + example: true + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server. Set to true for hosted control plane since API server runs in management cluster." + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: "networkExternalID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs in case there are several external networks." + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "networkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "dnsNameservers is the list of nameservers for the OpenStack Subnet being created. Set this value when you need to create a new network/subnet which requires access to DNS." + default: ["9.9.9.9", "149.112.112.112"] + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: "10.8.0.0/20" + example: "10.8.0.0/20" + description: |- + nodeCIDR is the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with nodeCIDR, + and a router connected to this subnet. + If you leave this empty, no network will be created. + # Workers + - name: workerFlavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.workerFlavor | quote }} + example: "SCS-4V-8" + description: "OpenStack instance flavor for worker nodes." + - name: workerRootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 25 + default: {{ .Values.workerRootDisk }} + description: |- + Root disk size in GiB for worker nodes. + OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. + Should be used for the diskless flavors (>= 20), otherwise set to 0. + - name: workerServerGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "869fe071-1e56-46a9-9166-47c9f228e297" + description: "The server group to assign the worker nodes to." + - name: workerAdditionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: [] + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: "" + example: "capi-keypair" + description: "The ssh key name to inject in the nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["security-group-1"] + description: |- + The names of extra security groups to assign to worker nodes. + Will be ignored if `securityGroupIDs` is used. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: [] + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: "The UUIDs of extra security groups to assign to worker nodes" + items: + type: string + - name: workerSecurityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["security-group-1"] + description: |- + The names of extra security groups to assign to the worker nodes. + Will be ignored if `workerSecurityGroupIDs` is used. + items: + type: string + - name: workerSecurityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: [] + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: "The UUIDs of extra security groups to assign to the worker nodes" + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {} + properties: + name: + type: string + example: "openstack" + default: "openstack" + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: "openstack" + description: "The name of the cloud to use from the clouds.yaml" + # Kubernetes API server + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: [] + example: ["mydomain.example"] + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: "none" + example: "none, octavia-amphora, octavia-ovn" + description: | + For hosted control plane, typically set to "none" since the API server runs in the management cluster. + You can choose from 3 options: + + none: + (default) No LoadBalancer solution will be deployed + + octavia-amphora: + Uses OpenStack's LoadBalancer service Octavia (provider:amphora) + + octavia-ovn: + Uses OpenStack's LoadBalancer service Octavia (provider:ovn) + - name: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs restrict access to the Kubernetes API server on a network level. + Ensure that at least the outgoing IP of your Management Cluster is added to the list of allowed CIDRs. + Otherwise CAPO can't reconcile the target Cluster correctly. + This requires amphora as load balancer provider in version >= v2.12. + items: + type: string + # + # Patches + # + patches: + # + # Patches for control plane variables (apply to HostedControlPlaneTemplate) + # + - name: controlPlaneReplicas + description: "Sets the number of hosted control plane replicas." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/replicas" + valueFrom: + variable: controlPlaneReplicas + - name: gatewayName + description: "Sets the Gateway API resource for control plane ingress." + enabledIf: {{ `'{{ ne .gatewayName "" }}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/gateway" + valueFrom: + template: | + name: {{ `{{ .gatewayName }}` }} + namespace: {{ `{{ .gatewayNamespace }}` }} + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: HostedControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServer/certSANs" + valueFrom: + variable: certSANs + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + description: "Takes care of the patches that should be applied when variable allowedCIDRs is set." + enabledIf: {{ `'{{ and .apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + # + # Patches for worker's OpenStackMachineTemplate resources. + # Note: No control plane patches since control plane runs in management cluster + # + - name: workerImage + description: "Sets the OpenStack image name that is used for creating the worker servers." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + - name: workerFlavor + description: "Sets the openstack instance flavor for the worker nodes." + enabledIf: {{ `'{{ ne .workerFlavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: workerFlavor + - name: workerRootDisk + description: "Sets the root disk size in GiB for worker nodes." + enabledIf: {{ `'{{ if .workerRootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .workerRootDisk }}` }} + - name: workerServerGroupID + description: "Sets the server group to assign the worker nodes to." + enabledIf: {{ `'{{ ne .workerServerGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .workerServerGroupID }}` }} + - name: workerAdditionalBlockDevices + enabledIf: {{ `'{{ if .workerAdditionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .workerAdditionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + # Note: The securityGroups patch must be placed before securityGroupIDs, workerSecurityGroups, and workerSecurityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: securityGroups + description: "Sets the list of the openstack security groups for the worker instances." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The securityGroupIDs patch must be placed before workerSecurityGroups, workerSecurityGroupIDs and after securityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: securityGroupIDs + description: "Sets the list of the openstack security groups for the worker instances by UUID." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the ssh key to inject in the nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # Note: The workerSecurityGroups patch must be placed before workerSecurityGroupIDs and after securityGroups and securityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroups + description: "Sets the list of the openstack security groups for the worker instances." + enabledIf: {{ `'{{ if .workerSecurityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .workerSecurityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + # Note: The workerSecurityGroupIDs patch must be placed after securityGroups, securityGroupIDs and workerSecurityGroupIDs. + # The patch order ensures the last applied patch overwrites previous ones. + - name: workerSecurityGroupIDs + description: "Sets the list of the openstack security groups for the worker instances by UUID." + enabledIf: {{ `'{{ if .workerSecurityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: false + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .workerSecurityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/hosted-control-plane-template.yaml b/providers/openstack/hcp/1-35/cluster-class/templates/hosted-control-plane-template.yaml new file mode 100644 index 00000000..b1caeea8 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/hosted-control-plane-template.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: HostedControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + gateway: + namespace: overridden-by-patch + name: overridden-by-patch diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/kubeadm-config-template-default-worker.yaml b/providers/openstack/hcp/1-35/cluster-class/templates/kubeadm-config-template-default-worker.yaml new file mode 100644 index 00000000..4c1494ed --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/kubeadm-config-template-default-worker.yaml @@ -0,0 +1,13 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/hcp/1-35/cluster-class/templates/openstack-cluster-template.yaml new file mode 100644 index 00000000..3b8ae609 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/openstack-cluster-template.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: + identityRef: + cloudName: overridden-by-patch + name: overridden-by-patch + apiServerLoadBalancer: + enabled: false + disableAPIServerFloatingIP: true + managedSecurityGroups: + allNodesSecurityGroupRules: + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: VXLAN (Cilium) + portRangeMin: 8472 + portRangeMax: 8472 + protocol: udp + description: "Allow VXLAN traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: HealthCheck (Cilium) + portRangeMin: 4240 + portRangeMax: 4240 + protocol: tcp + description: "Allow HealthCheck traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Hubble (Cilium) + portRangeMin: 4244 + portRangeMax: 4244 + protocol: tcp + description: "Allow Hubble traffic for Cilium" diff --git a/providers/openstack/hcp/1-35/cluster-class/templates/openstack-machine-template-default-worker.yaml b/providers/openstack/hcp/1-35/cluster-class/templates/openstack-machine-template-default-worker.yaml new file mode 100644 index 00000000..920dbb0a --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/templates/openstack-machine-template-default-worker.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/hcp/1-35/cluster-class/values.yaml b/providers/openstack/hcp/1-35/cluster-class/values.yaml new file mode 100644 index 00000000..5ebb9d61 --- /dev/null +++ b/providers/openstack/hcp/1-35/cluster-class/values.yaml @@ -0,0 +1,24 @@ +# Default values for hosted-control-plane cluster class + +# OpenStack credentials +identityRef: + name: "openstack" + cloudName: "openstack" + +# Network configuration +networkExternalID: "" +dnsNameservers: + - "9.9.9.9" + - "149.112.112.112" +nodeCIDR: "10.8.0.0/20" + +# Worker node configuration +workerFlavor: "SCS-2V-4" +workerRootDisk: 25 + +# Control plane configuration (hosted in management cluster) +controlPlaneReplicas: 3 + +# Gateway configuration for hosted control plane (optional) +gatewayName: "" +gatewayNamespace: "" diff --git a/providers/openstack/hcp/1-35/clusteraddon.yaml b/providers/openstack/hcp/1-35/clusteraddon.yaml new file mode 100644 index 00000000..d346ba22 --- /dev/null +++ b/providers/openstack/hcp/1-35/clusteraddon.yaml @@ -0,0 +1,21 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply diff --git a/providers/openstack/hcp/1-35/csctl.yaml b/providers/openstack/hcp/1-35/csctl.yaml new file mode 100644 index 00000000..cc70605d --- /dev/null +++ b/providers/openstack/hcp/1-35/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: hcp + kubernetesVersion: v1.35 + provider: + apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 + type: openstack diff --git a/providers/openstack/hcp/1-35/stack.yaml b/providers/openstack/hcp/1-35/stack.yaml new file mode 100644 index 00000000..4e3c1659 --- /dev/null +++ b/providers/openstack/hcp/1-35/stack.yaml @@ -0,0 +1,7 @@ +provider: openstack +clusterStackName: hcp +kubernetesVersion: 1.35 + +addons: + ccm: 2.35.x + csi: 2.35.x diff --git a/providers/openstack/scs/1-32/cluster-addon/ccm/Chart.yaml b/providers/openstack/scs/1-32/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..b66156ac --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.32.0 diff --git a/providers/openstack/scs/1-32/cluster-addon/ccm/overwrite.yaml b/providers/openstack/scs/1-32/cluster-addon/ccm/overwrite.yaml new file mode 100644 index 00000000..39076ecd --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/ccm/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + openstack-cloud-controller-manager: + cluster: + name: {{ .Cluster.metadata.name }} diff --git a/providers/openstack/scs/1-32/cluster-addon/ccm/values.yaml b/providers/openstack/scs/1-32/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/scs/1-32/cluster-addon/cni/Chart.yaml b/providers/openstack/scs/1-32/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..95516e98 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.2 diff --git a/providers/openstack/scs/1-32/cluster-addon/cni/values.yaml b/providers/openstack/scs/1-32/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/scs/1-32/cluster-addon/csi/Chart.yaml b/providers/openstack/scs/1-32/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..7d2ed6f0 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.32.2 diff --git a/providers/openstack/scs/1-32/cluster-addon/csi/overwrite.yaml b/providers/openstack/scs/1-32/cluster-addon/csi/overwrite.yaml new file mode 100644 index 00000000..d191a115 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/csi/overwrite.yaml @@ -0,0 +1,3 @@ +values: | + openstack-cinder-csi: + clusterID: "{{ .Cluster.metadata.name }}" diff --git a/providers/openstack/scs/1-32/cluster-addon/csi/values.yaml b/providers/openstack/scs/1-32/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/scs/1-32/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/scs/1-32/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/scs/1-32/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/scs/1-32/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/scs/1-32/cluster-addon/metrics-server/values.yaml b/providers/openstack/scs/1-32/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/scs/cluster-class/Chart.yaml b/providers/openstack/scs/1-32/cluster-class/Chart.yaml similarity index 68% rename from providers/openstack/scs/cluster-class/Chart.yaml rename to providers/openstack/scs/1-32/cluster-class/Chart.yaml index 642bef98..fa6d1f5d 100644 --- a/providers/openstack/scs/cluster-class/Chart.yaml +++ b/providers/openstack/scs/1-32/cluster-class/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 -description: 'This chart installs and configures: +description: "This chart installs and configures: * Openstack scs Cluster Class - ' + " name: openstack-scs-1-32-cluster-class type: application version: v1 diff --git a/providers/openstack/scs/1-32/cluster-class/templates/_helpers.tpl b/providers/openstack/scs/1-32/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/providers/openstack/scs/1-32/cluster-class/templates/cluster-class.yaml b/providers/openstack/scs/1-32/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..c9131e4c --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,863 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + machineInfrastructure: + templateRef: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5) + default: {{ .Values.variables.imageName | quote }} + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: {{ .Values.variables.imageIsOrc }} + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5. + default: {{ .Values.variables.imageAddVersion }} + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: |- + ID of an external OpenStack network. Required when multiple + external networks exist and VMs need public internet access. + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "MTU for the private cluster network. Set this to avoid fragmentation issues." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "DNS nameservers for the cluster subnet. Only used when a new network/subnet is created." + default: {{ .Values.variables.dnsNameservers | toJson }} + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: {{ .Values.variables.nodeCIDR | quote }} + example: "10.8.0.0/20" + description: |- + CIDR for the cluster subnet. CAPO will create a network, subnet, + and router. Leave empty to skip network creation. + # Machine configuration + # These apply to all nodes by default. Use topology.controlPlane.variables.overrides + # or topology.workers.machineDeployments[].variables.overrides to differentiate. + - name: flavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.flavor | quote }} + example: "SCS-2V-4" + description: |- + OpenStack instance flavor. Applies to all nodes by default. + Override per control plane or worker via topology variables overrides. + - name: rootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 50 + default: {{ .Values.variables.rootDisk }} + description: |- + Root disk size in GiB. OpenStack volume will be created and used + instead of an ephemeral disk defined in the flavor. + Set to 0 to use the flavor's ephemeral disk. + - name: serverGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.serverGroupID | quote }} + example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" + description: "Server group for anti-affinity placement. Override per CP/worker via topology." + - name: additionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.additionalBlockDevices | toJson }} + description: |- + Additional block devices (Cinder volumes) to attach to nodes. + Override per CP/worker via topology. + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Cluster-level (control plane only, managed by OpenStackClusterTemplate) + - name: controlPlaneAvailabilityZones + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.controlPlaneAvailabilityZones | toJson }} + example: ["nova"] + description: "Availability zones for control plane nodes (OpenStack cluster-level setting)." + items: + type: string + - name: controlPlaneOmitAvailabilityZone + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.controlPlaneOmitAvailabilityZone }} + description: |- + Omit availability zone when creating control plane nodes, letting the + Nova scheduler decide based on other scheduling constraints. + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.sshKeyName | quote }} + example: "capi-keypair" + description: "SSH key to inject into all nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.securityGroups | toJson }} + example: ["security-group-1"] + description: |- + Extra security groups by name for all nodes. + Ignored if securityGroupIDs is set. Override per CP/worker via topology. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: {{ .Values.variables.securityGroupIDs | toJson }} + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: |- + Extra security groups by UUID for all nodes. + Takes precedence over securityGroups. Override per CP/worker via topology. + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {{ .Values.variables.identityRef | toJson }} + properties: + name: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.name | quote }} + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.cloudName | quote }} + description: "The name of the cloud to use from the clouds.yaml" + # API server + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.disableAPIServerFloatingIP }} + description: "Disable the floating IP on the API server load balancer." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.certSANs | toJson }} + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.apiServerLoadBalancer | quote }} + example: "octavia-ovn" + description: |- + Load balancer in front of the API server. + Options: none, octavia-amphora, octavia-ovn (default). + - name: apiServerAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + Restrict access to the API server to these CIDRs (network-level ACL). + Requires amphora as load balancer provider (CAPO >= v2.12). + Ensure the management cluster's outgoing IP is included, + otherwise CAPO cannot reconcile the workload cluster. + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. This is typically the provider's discovery URL, changed to have an + empty path. + usernameClaim: + type: string + example: "preferred_username" + default: {{ .Values.variables.oidcConfig.usernameClaim | quote }} + description: >- + JWT claim to use as the user name. By default sub, + which is expected to be a unique identifier of the end user. Admins can choose + other claims, such as email or name, depending on their provider. However, + claims other than email will be prefixed with the issuer URL to prevent naming + clashes with other plugins. + groupsClaim: + type: string + example: "groups" + default: {{ .Values.variables.oidcConfig.groupsClaim | quote }} + description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." + usernamePrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.usernamePrefix | quote }} + description: >- + Prefix prepended to username claims to prevent + clashes with existing names (such as system: users). For example, the value + oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and + --oidc-username-claim is a value other than email the prefix defaults to ( + Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value + - can be used to disable all prefixing. + groupsPrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.groupsPrefix | quote }} + description: >- + Prefix prepended to group claims to prevent clashes + with existing names (such as system: groups). For example, the value oidc: will + create group names like oidc:engineering and oidc:infra. + # Container runtime + - name: registryMirrors + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.registryMirrors | toJson }} + description: "Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror." + items: + type: object + properties: + hostnameUpstream: + type: string + example: "docker.io" + description: "The hostname of the upstream registry." + urlUpstream: + type: string + example: "https://registry-1.docker.io" + description: "The server URL of the upstream registry." + urlMirror: + type: string + example: "https://registry.example.com/v2/dockerhub" + description: "The URL of the mirror registry." + certMirror: + type: string + example: "" + description: "TLS certificate of the mirror in PEM format (optional)." + # + # Patches + # + patches: + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerAllowedCIDRs + description: "Restricts API server access to the given CIDRs (requires amphora LB)." + enabledIf: {{ `'{{ and .apiServerAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: controlPlaneAvailabilityZones + description: "Sets the availability zones which control plane machines may be deployed to." + enabledIf: {{ `'{{ if .controlPlaneAvailabilityZones }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneAvailabilityZones" + valueFrom: + variable: controlPlaneAvailabilityZones + - name: controlPlaneOmitAvailabilityZone + description: "Causes availability zone to be omitted when creating control plane nodes." + enabledIf: {{ `'{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" + valueFrom: + variable: controlPlaneOmitAvailabilityZone + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + # + # Patches for OpenStackMachineTemplate resources (image). + # Image patches must stay separate because they use different builtin variables: + # - .builtin.controlPlane.version for CP + # - .builtin.machineDeployment.version for workers + # + - name: controlPlaneImage + description: "Sets the OpenStack image for control plane nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.controlPlane.version }}{{ end }}` }} + - name: workerImage + description: "Sets the OpenStack image for worker nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + # + # Unified machine patches — target both CP and workers. + # Users can override per CP/worker via topology.controlPlane.variables.overrides + # and topology.workers.machineDeployments[].variables.overrides. + # + - name: flavor + description: "Sets the OpenStack instance flavor for all nodes." + enabledIf: {{ `'{{ ne .flavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: flavor + - name: rootDisk + description: "Sets the root disk size in GiB. 0 means use ephemeral disk from flavor." + enabledIf: {{ `'{{ if .rootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .rootDisk }}` }} + - name: serverGroupID + description: "Sets the server group for anti-affinity." + enabledIf: {{ `'{{ ne .serverGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .serverGroupID }}` }} + - name: additionalBlockDevices + description: "Attaches additional Cinder volumes to nodes." + enabledIf: {{ `'{{ if .additionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .additionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + # + # Access patches — target both CP and workers. + # securityGroupIDs takes precedence over securityGroups (last patch wins). + # + - name: securityGroups + description: "Sets security groups by name for all nodes." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + - name: securityGroupIDs + description: "Sets security groups by UUID for all nodes. Takes precedence over securityGroups." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the SSH key to inject into all nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-client-id + value: {{ `'{{ .oidcConfig.clientID }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-issuer-url + value: {{ `'{{ .oidcConfig.issuerURL }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-claim + value: {{ `'{{ .oidcConfig.usernameClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-claim + value: {{ `'{{ .oidcConfig.groupsClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-prefix + value: {{ `'{{ .oidcConfig.usernamePrefix }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-prefix + value: {{ `'{{ .oidcConfig.groupsPrefix }}'` }} + # + # Registry mirror patches + # + - name: registryMirrorsControlPlane + description: "Configure registry mirrors on control plane nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} + - path: /etc/kube-proxy-config.yaml + owner: root:root + permissions: "0644" + content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + + - path: /etc/kube-proxy-patch.sh + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + kubeadm_file="/etc/kubeadm.yml" + [[ -f ${kubeadm_file} ]] || kubeadm_file="/run/kubeadm/kubeadm.yaml" + [[ -f ${kubeadm_file} ]] || exit 0 + [[ -f ${dir}/kube-proxy-config.yaml ]] || exit 0 + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm -f "${dir}/kube-proxy-config.yaml" + echo success > /tmp/kube-proxy-patch + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-" + value: "bash /etc/kube-proxy-patch.sh" + - name: registryMirrorsWorker + description: "Configure registry mirrors on worker nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} diff --git a/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml new file mode 100644 index 00000000..732d78b3 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..b199f36c --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,109 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + - name: cloud-provider + value: external + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10257" + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "100" + scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10259" + - name: profiling + value: "false" + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 + - name: auto-compaction-mode + value: periodic + - name: auto-compaction-retention + value: 8h + - name: election-timeout + value: "2500" + - name: heartbeat-interval + value: "250" + - name: snapshot-count + value: "6400" + files: + - content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + path: /etc/kube-proxy-config.yaml + - content: | + #!/usr/bin/env bash + + # + # (PK) I couldn't find a better/simpler way to conifgure it. See: + # https://github.com/kubernetes-sigs/cluster-api/issues/4512 + # + + set -o errexit + set -o nounset + set -o pipefail + + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + readonly dir + + # Exit fast if already appended. + if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then + exit 0 + fi + + # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). + kubeadm_file="/etc/kubeadm.yml" + if [[ ! -f ${kubeadm_file} ]]; then + kubeadm_file="/run/kubeadm/kubeadm.yaml" + fi + + # Run this script only if this is the init node. + if [[ ! -f ${kubeadm_file} ]]; then + exit 0 + fi + + # Append kube-proxy-config.yaml to kubeadm config and delete it + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm "${dir}/kube-proxy-config.yaml" + + echo success > /tmp/kube-proxy-patch + owner: root:root + path: /etc/kube-proxy-patch.sh + permissions: "0755" + preKubeadmCommands: + - bash /etc/kube-proxy-patch.sh + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-32/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/scs/1-32/cluster-class/templates/openstack-cluster-template.yaml new file mode 100644 index 00000000..3cc03fbb --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/openstack-cluster-template.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: + identityRef: + cloudName: overridden-by-patch + name: overridden-by-patch + apiServerLoadBalancer: + enabled: false + managedSecurityGroups: + allNodesSecurityGroupRules: + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: VXLAN (Cilium) + portRangeMin: 8472 + portRangeMax: 8472 + protocol: udp + description: "Allow VXLAN traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: HealthCheck (Cilium) + portRangeMin: 4240 + portRangeMax: 4240 + protocol: tcp + description: "Allow HealthCheck traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: Hubble (Cilium) + portRangeMin: 4244 + portRangeMax: 4244 + protocol: tcp + description: "Allow Hubble traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-proxy exporter + portRangeMin: 10249 + portRangeMax: 10249 + protocol: tcp + description: "Allow Prometheus traffic for kube-proxy exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-controller-manager exporter + portRangeMin: 10257 + portRangeMax: 10257 + protocol: tcp + description: "Allow Prometheus traffic for kube-controller-manager exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-scheduler exporter + portRangeMin: 10259 + portRangeMax: 10259 + protocol: tcp + description: "Allow Prometheus traffic for kube-scheduler exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus node exporter + portRangeMin: 9100 + portRangeMax: 9100 + protocol: tcp + description: "Allow Prometheus traffic for scraping node exporter" + controlPlaneNodesSecurityGroupRules: + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus etcd exporter + portRangeMin: 2381 + portRangeMax: 2381 + protocol: tcp + description: "Allow Prometheus traffic for scraping etcd exporter" \ No newline at end of file diff --git a/providers/openstack/scs2/cluster-class/templates/openstack-machine-template-control-plane.yaml b/providers/openstack/scs/1-32/cluster-class/templates/openstack-machine-template-control-plane.yaml similarity index 100% rename from providers/openstack/scs2/cluster-class/templates/openstack-machine-template-control-plane.yaml rename to providers/openstack/scs/1-32/cluster-class/templates/openstack-machine-template-control-plane.yaml diff --git a/providers/openstack/scs/1-32/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/scs/1-32/cluster-class/templates/openstack-machine-template-worker.yaml new file mode 100644 index 00000000..920dbb0a --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/templates/openstack-machine-template-worker.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-32/cluster-class/values.yaml b/providers/openstack/scs/1-32/cluster-class/values.yaml new file mode 100644 index 00000000..1344cf85 --- /dev/null +++ b/providers/openstack/scs/1-32/cluster-class/values.yaml @@ -0,0 +1,50 @@ +# ClusterClass variable defaults +# These are referenced by the ClusterClass template and can be overridden per deployment. +# Variables apply to all nodes by default. Use topology.controlPlane.variables.overrides +# or topology.workers.machineDeployments[].variables.overrides to differentiate. +variables: + # Image configuration + imageName: "ubuntu-capi-image" + imageIsOrc: false + imageAddVersion: true + + # API server + disableAPIServerFloatingIP: false + apiServerLoadBalancer: "octavia-ovn" + certSANs: [] + + # Network + dnsNameservers: ["9.9.9.9", "149.112.112.112"] + nodeCIDR: "10.8.0.0/20" + + # Machine configuration (override per CP/worker via topology) + flavor: "SCS-2V-4" + rootDisk: 50 + serverGroupID: "" + additionalBlockDevices: [] + + # Access management + sshKeyName: "" + securityGroups: [] + securityGroupIDs: [] + + # Cluster-level (control plane only) + controlPlaneAvailabilityZones: [] + controlPlaneOmitAvailabilityZone: false + + # Identity + identityRef: + name: "openstack" + cloudName: "openstack" + + # Container runtime + registryMirrors: [] + + # OIDC + oidcConfig: + clientID: "" + issuerURL: "" + usernameClaim: "preferred_username" + groupsClaim: "groups" + usernamePrefix: "oidc:" + groupsPrefix: "oidc:" diff --git a/providers/openstack/scs/1-32/clusteraddon.yaml b/providers/openstack/scs/1-32/clusteraddon.yaml new file mode 100644 index 00000000..bea5fc78 --- /dev/null +++ b/providers/openstack/scs/1-32/clusteraddon.yaml @@ -0,0 +1,30 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply diff --git a/providers/openstack/scs/csctl.yaml b/providers/openstack/scs/1-32/csctl.yaml similarity index 85% rename from providers/openstack/scs/csctl.yaml rename to providers/openstack/scs/1-32/csctl.yaml index 7a65ae35..2b25389b 100644 --- a/providers/openstack/scs/csctl.yaml +++ b/providers/openstack/scs/1-32/csctl.yaml @@ -1,7 +1,7 @@ apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 config: clusterStackName: scs - kubernetesVersion: v1.32.5 + kubernetesVersion: v1.32 provider: apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 type: openstack diff --git a/providers/openstack/scs/1-32/stack.yaml b/providers/openstack/scs/1-32/stack.yaml new file mode 100644 index 00000000..690ec6c5 --- /dev/null +++ b/providers/openstack/scs/1-32/stack.yaml @@ -0,0 +1,6 @@ +provider: openstack +clusterStackName: scs +kubernetesVersion: 1.32.13 +addons: + ccm: 2.32.x + csi: 2.32.x diff --git a/providers/openstack/scs/1-33/cluster-addon/ccm/Chart.yaml b/providers/openstack/scs/1-33/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..541cecd0 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.33.1 diff --git a/providers/openstack/scs/1-33/cluster-addon/ccm/overwrite.yaml b/providers/openstack/scs/1-33/cluster-addon/ccm/overwrite.yaml new file mode 100644 index 00000000..39076ecd --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/ccm/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + openstack-cloud-controller-manager: + cluster: + name: {{ .Cluster.metadata.name }} diff --git a/providers/openstack/scs/1-33/cluster-addon/ccm/values.yaml b/providers/openstack/scs/1-33/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/scs/1-33/cluster-addon/cni/Chart.yaml b/providers/openstack/scs/1-33/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..95516e98 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.2 diff --git a/providers/openstack/scs/1-33/cluster-addon/cni/values.yaml b/providers/openstack/scs/1-33/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/scs/1-33/cluster-addon/csi/Chart.yaml b/providers/openstack/scs/1-33/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..e275b2ff --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.33.1 diff --git a/providers/openstack/scs/1-33/cluster-addon/csi/overwrite.yaml b/providers/openstack/scs/1-33/cluster-addon/csi/overwrite.yaml new file mode 100644 index 00000000..d191a115 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/csi/overwrite.yaml @@ -0,0 +1,3 @@ +values: | + openstack-cinder-csi: + clusterID: "{{ .Cluster.metadata.name }}" diff --git a/providers/openstack/scs/1-33/cluster-addon/csi/values.yaml b/providers/openstack/scs/1-33/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/scs/1-33/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/scs/1-33/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/scs/1-33/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/scs/1-33/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/scs/1-33/cluster-addon/metrics-server/values.yaml b/providers/openstack/scs/1-33/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/scs2/cluster-class/Chart.yaml b/providers/openstack/scs/1-33/cluster-class/Chart.yaml similarity index 58% rename from providers/openstack/scs2/cluster-class/Chart.yaml rename to providers/openstack/scs/1-33/cluster-class/Chart.yaml index 2ea03e47..82b14c74 100644 --- a/providers/openstack/scs2/cluster-class/Chart.yaml +++ b/providers/openstack/scs/1-33/cluster-class/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 description: "This chart installs and configures: - * Openstack scs2 Cluster Class + * Openstack scs Cluster Class " -name: openstack-scs2-1-34-cluster-class +name: openstack-scs-1-33-cluster-class type: application version: v1 diff --git a/providers/openstack/scs/1-33/cluster-class/templates/_helpers.tpl b/providers/openstack/scs/1-33/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/providers/openstack/scs/1-33/cluster-class/templates/cluster-class.yaml b/providers/openstack/scs/1-33/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..70eab050 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,868 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + machineInfrastructure: + templateRef: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5) + default: {{ .Values.variables.imageName | quote }} + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: {{ .Values.variables.imageIsOrc }} + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5. + default: {{ .Values.variables.imageAddVersion }} + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: |- + ID of an external OpenStack network. Required when multiple + external networks exist and VMs need public internet access. + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "MTU for the private cluster network. Set this to avoid fragmentation issues." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "DNS nameservers for the cluster subnet. Only used when a new network/subnet is created." + default: {{ .Values.variables.dnsNameservers | toJson }} + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: {{ .Values.variables.nodeCIDR | quote }} + example: "10.8.0.0/20" + description: |- + CIDR for the cluster subnet. CAPO will create a network, subnet, + and router. Leave empty to skip network creation. + # Machine configuration + # These apply to all nodes by default. Use topology.controlPlane.variables.overrides + # or topology.workers.machineDeployments[].variables.overrides to differentiate. + - name: flavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.flavor | quote }} + example: "SCS-2V-4" + description: |- + OpenStack instance flavor. Applies to all nodes by default. + Override per control plane or worker via topology variables overrides. + - name: rootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 50 + default: {{ .Values.variables.rootDisk }} + description: |- + Root disk size in GiB. OpenStack volume will be created and used + instead of an ephemeral disk defined in the flavor. + Set to 0 to use the flavor's ephemeral disk. + - name: serverGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.serverGroupID | quote }} + example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" + description: "Server group for anti-affinity placement. Override per CP/worker via topology." + - name: additionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.additionalBlockDevices | toJson }} + description: |- + Additional block devices (Cinder volumes) to attach to nodes. + Override per CP/worker via topology. + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Cluster-level (control plane only, managed by OpenStackClusterTemplate) + - name: controlPlaneAvailabilityZones + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.controlPlaneAvailabilityZones | toJson }} + example: ["nova"] + description: "Availability zones for control plane nodes (OpenStack cluster-level setting)." + items: + type: string + - name: controlPlaneOmitAvailabilityZone + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.controlPlaneOmitAvailabilityZone }} + description: |- + Omit availability zone when creating control plane nodes, letting the + Nova scheduler decide based on other scheduling constraints. + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.sshKeyName | quote }} + example: "capi-keypair" + description: "SSH key to inject into all nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.securityGroups | toJson }} + example: ["security-group-1"] + description: |- + Extra security groups by name for all nodes. + Ignored if securityGroupIDs is set. Override per CP/worker via topology. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: {{ .Values.variables.securityGroupIDs | toJson }} + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: |- + Extra security groups by UUID for all nodes. + Takes precedence over securityGroups. Override per CP/worker via topology. + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {{ .Values.variables.identityRef | toJson }} + properties: + name: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.name | quote }} + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.cloudName | quote }} + description: "The name of the cloud to use from the clouds.yaml" + type: + type: string + example: "Secret" + default: {{ .Values.variables.identityRef.type | quote }} + description: "The type of the identityRef" + # API server + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.disableAPIServerFloatingIP }} + description: "Disable the floating IP on the API server load balancer." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.certSANs | toJson }} + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.apiServerLoadBalancer | quote }} + example: "octavia-ovn" + description: |- + Load balancer in front of the API server. + Options: none, octavia-amphora, octavia-ovn (default). + - name: apiServerAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + Restrict access to the API server to these CIDRs (network-level ACL). + Requires amphora as load balancer provider (CAPO >= v2.12). + Ensure the management cluster's outgoing IP is included, + otherwise CAPO cannot reconcile the workload cluster. + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. This is typically the provider's discovery URL, changed to have an + empty path. + usernameClaim: + type: string + example: "preferred_username" + default: {{ .Values.variables.oidcConfig.usernameClaim | quote }} + description: >- + JWT claim to use as the user name. By default sub, + which is expected to be a unique identifier of the end user. Admins can choose + other claims, such as email or name, depending on their provider. However, + claims other than email will be prefixed with the issuer URL to prevent naming + clashes with other plugins. + groupsClaim: + type: string + example: "groups" + default: {{ .Values.variables.oidcConfig.groupsClaim | quote }} + description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." + usernamePrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.usernamePrefix | quote }} + description: >- + Prefix prepended to username claims to prevent + clashes with existing names (such as system: users). For example, the value + oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and + --oidc-username-claim is a value other than email the prefix defaults to ( + Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value + - can be used to disable all prefixing. + groupsPrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.groupsPrefix | quote }} + description: >- + Prefix prepended to group claims to prevent clashes + with existing names (such as system: groups). For example, the value oidc: will + create group names like oidc:engineering and oidc:infra. + # Container runtime + - name: registryMirrors + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.registryMirrors | toJson }} + description: "Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror." + items: + type: object + properties: + hostnameUpstream: + type: string + example: "docker.io" + description: "The hostname of the upstream registry." + urlUpstream: + type: string + example: "https://registry-1.docker.io" + description: "The server URL of the upstream registry." + urlMirror: + type: string + example: "https://registry.example.com/v2/dockerhub" + description: "The URL of the mirror registry." + certMirror: + type: string + example: "" + description: "TLS certificate of the mirror in PEM format (optional)." + # + # Patches + # + patches: + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerAllowedCIDRs + description: "Restricts API server access to the given CIDRs (requires amphora LB)." + enabledIf: {{ `'{{ and .apiServerAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: controlPlaneAvailabilityZones + description: "Sets the availability zones which control plane machines may be deployed to." + enabledIf: {{ `'{{ if .controlPlaneAvailabilityZones }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneAvailabilityZones" + valueFrom: + variable: controlPlaneAvailabilityZones + - name: controlPlaneOmitAvailabilityZone + description: "Causes availability zone to be omitted when creating control plane nodes." + enabledIf: {{ `'{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" + valueFrom: + variable: controlPlaneOmitAvailabilityZone + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + # + # Patches for OpenStackMachineTemplate resources (image). + # Image patches must stay separate because they use different builtin variables: + # - .builtin.controlPlane.version for CP + # - .builtin.machineDeployment.version for workers + # + - name: controlPlaneImage + description: "Sets the OpenStack image for control plane nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.controlPlane.version }}{{ end }}` }} + - name: workerImage + description: "Sets the OpenStack image for worker nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + # + # Unified machine patches — target both CP and workers. + # Users can override per CP/worker via topology.controlPlane.variables.overrides + # and topology.workers.machineDeployments[].variables.overrides. + # + - name: flavor + description: "Sets the OpenStack instance flavor for all nodes." + enabledIf: {{ `'{{ ne .flavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: flavor + - name: rootDisk + description: "Sets the root disk size in GiB. 0 means use ephemeral disk from flavor." + enabledIf: {{ `'{{ if .rootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .rootDisk }}` }} + - name: serverGroupID + description: "Sets the server group for anti-affinity." + enabledIf: {{ `'{{ ne .serverGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .serverGroupID }}` }} + - name: additionalBlockDevices + description: "Attaches additional Cinder volumes to nodes." + enabledIf: {{ `'{{ if .additionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .additionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + # + # Access patches — target both CP and workers. + # securityGroupIDs takes precedence over securityGroups (last patch wins). + # + - name: securityGroups + description: "Sets security groups by name for all nodes." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + - name: securityGroupIDs + description: "Sets security groups by UUID for all nodes. Takes precedence over securityGroups." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the SSH key to inject into all nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-client-id + value: {{ `'{{ .oidcConfig.clientID }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-issuer-url + value: {{ `'{{ .oidcConfig.issuerURL }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-claim + value: {{ `'{{ .oidcConfig.usernameClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-claim + value: {{ `'{{ .oidcConfig.groupsClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-prefix + value: {{ `'{{ .oidcConfig.usernamePrefix }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-prefix + value: {{ `'{{ .oidcConfig.groupsPrefix }}'` }} + # + # Registry mirror patches + # + - name: registryMirrorsControlPlane + description: "Configure registry mirrors on control plane nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} + - path: /etc/kube-proxy-config.yaml + owner: root:root + permissions: "0644" + content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + + - path: /etc/kube-proxy-patch.sh + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + kubeadm_file="/etc/kubeadm.yml" + [[ -f ${kubeadm_file} ]] || kubeadm_file="/run/kubeadm/kubeadm.yaml" + [[ -f ${kubeadm_file} ]] || exit 0 + [[ -f ${dir}/kube-proxy-config.yaml ]] || exit 0 + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm -f "${dir}/kube-proxy-config.yaml" + echo success > /tmp/kube-proxy-patch + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-" + value: "bash /etc/kube-proxy-patch.sh" + - name: registryMirrorsWorker + description: "Configure registry mirrors on worker nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} diff --git a/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml new file mode 100644 index 00000000..732d78b3 --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..b199f36c --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,109 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + - name: cloud-provider + value: external + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10257" + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "100" + scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10259" + - name: profiling + value: "false" + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 + - name: auto-compaction-mode + value: periodic + - name: auto-compaction-retention + value: 8h + - name: election-timeout + value: "2500" + - name: heartbeat-interval + value: "250" + - name: snapshot-count + value: "6400" + files: + - content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + path: /etc/kube-proxy-config.yaml + - content: | + #!/usr/bin/env bash + + # + # (PK) I couldn't find a better/simpler way to conifgure it. See: + # https://github.com/kubernetes-sigs/cluster-api/issues/4512 + # + + set -o errexit + set -o nounset + set -o pipefail + + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + readonly dir + + # Exit fast if already appended. + if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then + exit 0 + fi + + # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). + kubeadm_file="/etc/kubeadm.yml" + if [[ ! -f ${kubeadm_file} ]]; then + kubeadm_file="/run/kubeadm/kubeadm.yaml" + fi + + # Run this script only if this is the init node. + if [[ ! -f ${kubeadm_file} ]]; then + exit 0 + fi + + # Append kube-proxy-config.yaml to kubeadm config and delete it + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm "${dir}/kube-proxy-config.yaml" + + echo success > /tmp/kube-proxy-patch + owner: root:root + path: /etc/kube-proxy-patch.sh + permissions: "0755" + preKubeadmCommands: + - bash /etc/kube-proxy-patch.sh + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-33/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/scs/1-33/cluster-class/templates/openstack-cluster-template.yaml new file mode 100644 index 00000000..6d96089f --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/openstack-cluster-template.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: + identityRef: + cloudName: overridden-by-patch + name: overridden-by-patch + apiServerLoadBalancer: + enabled: false + managedSecurityGroups: + allNodesSecurityGroupRules: + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: VXLAN (Cilium) + portRangeMin: 8472 + portRangeMax: 8472 + protocol: udp + description: "Allow VXLAN traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: HealthCheck (Cilium) + portRangeMin: 4240 + portRangeMax: 4240 + protocol: tcp + description: "Allow HealthCheck traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: Hubble (Cilium) + portRangeMin: 4244 + portRangeMax: 4244 + protocol: tcp + description: "Allow Hubble traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-proxy exporter + portRangeMin: 10249 + portRangeMax: 10249 + protocol: tcp + description: "Allow Prometheus traffic for kube-proxy exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-controller-manager exporter + portRangeMin: 10257 + portRangeMax: 10257 + protocol: tcp + description: "Allow Prometheus traffic for kube-controller-manager exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-scheduler exporter + portRangeMin: 10259 + portRangeMax: 10259 + protocol: tcp + description: "Allow Prometheus traffic for kube-scheduler exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus node exporter + portRangeMin: 9100 + portRangeMax: 9100 + protocol: tcp + description: "Allow Prometheus traffic for scraping node exporter" + controlPlaneNodesSecurityGroupRules: + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus etcd exporter + portRangeMin: 2381 + portRangeMax: 2381 + protocol: tcp + description: "Allow Prometheus traffic for scraping etcd exporter" diff --git a/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-control-plane.yaml b/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-control-plane.yaml new file mode 100644 index 00000000..703c1b1c --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-control-plane.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-worker.yaml new file mode 100644 index 00000000..920dbb0a --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/templates/openstack-machine-template-worker.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-33/cluster-class/values.yaml b/providers/openstack/scs/1-33/cluster-class/values.yaml new file mode 100644 index 00000000..272a7a0a --- /dev/null +++ b/providers/openstack/scs/1-33/cluster-class/values.yaml @@ -0,0 +1,51 @@ +# ClusterClass variable defaults +# These are referenced by the ClusterClass template and can be overridden per deployment. +# Variables apply to all nodes by default. Use topology.controlPlane.variables.overrides +# or topology.workers.machineDeployments[].variables.overrides to differentiate. +variables: + # Image configuration + imageName: "ubuntu-capi-image" + imageIsOrc: false + imageAddVersion: true + + # API server + disableAPIServerFloatingIP: false + apiServerLoadBalancer: "octavia-ovn" + certSANs: [] + + # Network + dnsNameservers: ["9.9.9.9", "149.112.112.112"] + nodeCIDR: "10.8.0.0/20" + + # Machine configuration (override per CP/worker via topology) + flavor: "SCS-2V-4" + rootDisk: 50 + serverGroupID: "" + additionalBlockDevices: [] + + # Access management + sshKeyName: "" + securityGroups: [] + securityGroupIDs: [] + + # Cluster-level (control plane only) + controlPlaneAvailabilityZones: [] + controlPlaneOmitAvailabilityZone: false + + # Identity + identityRef: + name: "openstack" + cloudName: "openstack" + type: "Secret" + + # Container runtime + registryMirrors: [] + + # OIDC + oidcConfig: + clientID: "" + issuerURL: "" + usernameClaim: "preferred_username" + groupsClaim: "groups" + usernamePrefix: "oidc:" + groupsPrefix: "oidc:" diff --git a/providers/openstack/scs/1-33/clusteraddon.yaml b/providers/openstack/scs/1-33/clusteraddon.yaml new file mode 100644 index 00000000..bea5fc78 --- /dev/null +++ b/providers/openstack/scs/1-33/clusteraddon.yaml @@ -0,0 +1,30 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply diff --git a/providers/openstack/scs/1-33/csctl.yaml b/providers/openstack/scs/1-33/csctl.yaml new file mode 100644 index 00000000..f468ad56 --- /dev/null +++ b/providers/openstack/scs/1-33/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.33 + provider: + apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 + type: openstack diff --git a/providers/openstack/scs/1-33/stack.yaml b/providers/openstack/scs/1-33/stack.yaml new file mode 100644 index 00000000..9047a30b --- /dev/null +++ b/providers/openstack/scs/1-33/stack.yaml @@ -0,0 +1,6 @@ +provider: openstack +clusterStackName: scs +kubernetesVersion: 1.33.10 +addons: + ccm: 2.33.x + csi: 2.33.x diff --git a/providers/openstack/scs/1-34/cluster-addon/ccm/Chart.yaml b/providers/openstack/scs/1-34/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..bd086402 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.34.2 diff --git a/providers/openstack/scs/1-34/cluster-addon/ccm/overwrite.yaml b/providers/openstack/scs/1-34/cluster-addon/ccm/overwrite.yaml new file mode 100644 index 00000000..39076ecd --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/ccm/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + openstack-cloud-controller-manager: + cluster: + name: {{ .Cluster.metadata.name }} diff --git a/providers/openstack/scs/1-34/cluster-addon/ccm/values.yaml b/providers/openstack/scs/1-34/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/scs/1-34/cluster-addon/cni/Chart.yaml b/providers/openstack/scs/1-34/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..95516e98 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.2 diff --git a/providers/openstack/scs/1-34/cluster-addon/cni/values.yaml b/providers/openstack/scs/1-34/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..8a312f0c --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/cni/values.yaml @@ -0,0 +1,14 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + ipam: + mode: "kubernetes" + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/scs/1-34/cluster-addon/csi/Chart.yaml b/providers/openstack/scs/1-34/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..cbe0cc17 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.34.3 diff --git a/providers/openstack/scs/1-34/cluster-addon/csi/overwrite.yaml b/providers/openstack/scs/1-34/cluster-addon/csi/overwrite.yaml new file mode 100644 index 00000000..d191a115 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/csi/overwrite.yaml @@ -0,0 +1,3 @@ +values: | + openstack-cinder-csi: + clusterID: "{{ .Cluster.metadata.name }}" diff --git a/providers/openstack/scs/1-34/cluster-addon/csi/values.yaml b/providers/openstack/scs/1-34/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/scs/1-34/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/scs/1-34/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/scs/1-34/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/scs/1-34/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/scs/1-34/cluster-addon/metrics-server/values.yaml b/providers/openstack/scs/1-34/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/scs/1-34/cluster-class/Chart.yaml b/providers/openstack/scs/1-34/cluster-class/Chart.yaml new file mode 100644 index 00000000..4fa495ee --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +description: "This chart installs and configures: + + * Openstack scs Cluster Class + + " +name: openstack-scs-1-34-cluster-class +type: application +version: v1 diff --git a/providers/openstack/scs/1-34/cluster-class/templates/_helpers.tpl b/providers/openstack/scs/1-34/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/providers/openstack/scs/1-34/cluster-class/templates/cluster-class.yaml b/providers/openstack/scs/1-34/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..c9131e4c --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,863 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + machineInfrastructure: + templateRef: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5) + default: {{ .Values.variables.imageName | quote }} + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: {{ .Values.variables.imageIsOrc }} + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5. + default: {{ .Values.variables.imageAddVersion }} + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: |- + ID of an external OpenStack network. Required when multiple + external networks exist and VMs need public internet access. + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "MTU for the private cluster network. Set this to avoid fragmentation issues." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "DNS nameservers for the cluster subnet. Only used when a new network/subnet is created." + default: {{ .Values.variables.dnsNameservers | toJson }} + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: {{ .Values.variables.nodeCIDR | quote }} + example: "10.8.0.0/20" + description: |- + CIDR for the cluster subnet. CAPO will create a network, subnet, + and router. Leave empty to skip network creation. + # Machine configuration + # These apply to all nodes by default. Use topology.controlPlane.variables.overrides + # or topology.workers.machineDeployments[].variables.overrides to differentiate. + - name: flavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.flavor | quote }} + example: "SCS-2V-4" + description: |- + OpenStack instance flavor. Applies to all nodes by default. + Override per control plane or worker via topology variables overrides. + - name: rootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 50 + default: {{ .Values.variables.rootDisk }} + description: |- + Root disk size in GiB. OpenStack volume will be created and used + instead of an ephemeral disk defined in the flavor. + Set to 0 to use the flavor's ephemeral disk. + - name: serverGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.serverGroupID | quote }} + example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" + description: "Server group for anti-affinity placement. Override per CP/worker via topology." + - name: additionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.additionalBlockDevices | toJson }} + description: |- + Additional block devices (Cinder volumes) to attach to nodes. + Override per CP/worker via topology. + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Cluster-level (control plane only, managed by OpenStackClusterTemplate) + - name: controlPlaneAvailabilityZones + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.controlPlaneAvailabilityZones | toJson }} + example: ["nova"] + description: "Availability zones for control plane nodes (OpenStack cluster-level setting)." + items: + type: string + - name: controlPlaneOmitAvailabilityZone + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.controlPlaneOmitAvailabilityZone }} + description: |- + Omit availability zone when creating control plane nodes, letting the + Nova scheduler decide based on other scheduling constraints. + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.sshKeyName | quote }} + example: "capi-keypair" + description: "SSH key to inject into all nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.securityGroups | toJson }} + example: ["security-group-1"] + description: |- + Extra security groups by name for all nodes. + Ignored if securityGroupIDs is set. Override per CP/worker via topology. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: {{ .Values.variables.securityGroupIDs | toJson }} + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: |- + Extra security groups by UUID for all nodes. + Takes precedence over securityGroups. Override per CP/worker via topology. + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {{ .Values.variables.identityRef | toJson }} + properties: + name: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.name | quote }} + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.cloudName | quote }} + description: "The name of the cloud to use from the clouds.yaml" + # API server + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.disableAPIServerFloatingIP }} + description: "Disable the floating IP on the API server load balancer." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.certSANs | toJson }} + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.apiServerLoadBalancer | quote }} + example: "octavia-ovn" + description: |- + Load balancer in front of the API server. + Options: none, octavia-amphora, octavia-ovn (default). + - name: apiServerAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + Restrict access to the API server to these CIDRs (network-level ACL). + Requires amphora as load balancer provider (CAPO >= v2.12). + Ensure the management cluster's outgoing IP is included, + otherwise CAPO cannot reconcile the workload cluster. + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. This is typically the provider's discovery URL, changed to have an + empty path. + usernameClaim: + type: string + example: "preferred_username" + default: {{ .Values.variables.oidcConfig.usernameClaim | quote }} + description: >- + JWT claim to use as the user name. By default sub, + which is expected to be a unique identifier of the end user. Admins can choose + other claims, such as email or name, depending on their provider. However, + claims other than email will be prefixed with the issuer URL to prevent naming + clashes with other plugins. + groupsClaim: + type: string + example: "groups" + default: {{ .Values.variables.oidcConfig.groupsClaim | quote }} + description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." + usernamePrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.usernamePrefix | quote }} + description: >- + Prefix prepended to username claims to prevent + clashes with existing names (such as system: users). For example, the value + oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and + --oidc-username-claim is a value other than email the prefix defaults to ( + Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value + - can be used to disable all prefixing. + groupsPrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.groupsPrefix | quote }} + description: >- + Prefix prepended to group claims to prevent clashes + with existing names (such as system: groups). For example, the value oidc: will + create group names like oidc:engineering and oidc:infra. + # Container runtime + - name: registryMirrors + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.registryMirrors | toJson }} + description: "Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror." + items: + type: object + properties: + hostnameUpstream: + type: string + example: "docker.io" + description: "The hostname of the upstream registry." + urlUpstream: + type: string + example: "https://registry-1.docker.io" + description: "The server URL of the upstream registry." + urlMirror: + type: string + example: "https://registry.example.com/v2/dockerhub" + description: "The URL of the mirror registry." + certMirror: + type: string + example: "" + description: "TLS certificate of the mirror in PEM format (optional)." + # + # Patches + # + patches: + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerAllowedCIDRs + description: "Restricts API server access to the given CIDRs (requires amphora LB)." + enabledIf: {{ `'{{ and .apiServerAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: controlPlaneAvailabilityZones + description: "Sets the availability zones which control plane machines may be deployed to." + enabledIf: {{ `'{{ if .controlPlaneAvailabilityZones }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneAvailabilityZones" + valueFrom: + variable: controlPlaneAvailabilityZones + - name: controlPlaneOmitAvailabilityZone + description: "Causes availability zone to be omitted when creating control plane nodes." + enabledIf: {{ `'{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" + valueFrom: + variable: controlPlaneOmitAvailabilityZone + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + # + # Patches for OpenStackMachineTemplate resources (image). + # Image patches must stay separate because they use different builtin variables: + # - .builtin.controlPlane.version for CP + # - .builtin.machineDeployment.version for workers + # + - name: controlPlaneImage + description: "Sets the OpenStack image for control plane nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.controlPlane.version }}{{ end }}` }} + - name: workerImage + description: "Sets the OpenStack image for worker nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + # + # Unified machine patches — target both CP and workers. + # Users can override per CP/worker via topology.controlPlane.variables.overrides + # and topology.workers.machineDeployments[].variables.overrides. + # + - name: flavor + description: "Sets the OpenStack instance flavor for all nodes." + enabledIf: {{ `'{{ ne .flavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: flavor + - name: rootDisk + description: "Sets the root disk size in GiB. 0 means use ephemeral disk from flavor." + enabledIf: {{ `'{{ if .rootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .rootDisk }}` }} + - name: serverGroupID + description: "Sets the server group for anti-affinity." + enabledIf: {{ `'{{ ne .serverGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .serverGroupID }}` }} + - name: additionalBlockDevices + description: "Attaches additional Cinder volumes to nodes." + enabledIf: {{ `'{{ if .additionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .additionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + # + # Access patches — target both CP and workers. + # securityGroupIDs takes precedence over securityGroups (last patch wins). + # + - name: securityGroups + description: "Sets security groups by name for all nodes." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + - name: securityGroupIDs + description: "Sets security groups by UUID for all nodes. Takes precedence over securityGroups." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the SSH key to inject into all nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-client-id + value: {{ `'{{ .oidcConfig.clientID }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-issuer-url + value: {{ `'{{ .oidcConfig.issuerURL }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-claim + value: {{ `'{{ .oidcConfig.usernameClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-claim + value: {{ `'{{ .oidcConfig.groupsClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-prefix + value: {{ `'{{ .oidcConfig.usernamePrefix }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-prefix + value: {{ `'{{ .oidcConfig.groupsPrefix }}'` }} + # + # Registry mirror patches + # + - name: registryMirrorsControlPlane + description: "Configure registry mirrors on control plane nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} + - path: /etc/kube-proxy-config.yaml + owner: root:root + permissions: "0644" + content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + + - path: /etc/kube-proxy-patch.sh + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + kubeadm_file="/etc/kubeadm.yml" + [[ -f ${kubeadm_file} ]] || kubeadm_file="/run/kubeadm/kubeadm.yaml" + [[ -f ${kubeadm_file} ]] || exit 0 + [[ -f ${dir}/kube-proxy-config.yaml ]] || exit 0 + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm -f "${dir}/kube-proxy-config.yaml" + echo success > /tmp/kube-proxy-patch + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-" + value: "bash /etc/kube-proxy-patch.sh" + - name: registryMirrorsWorker + description: "Configure registry mirrors on worker nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} diff --git a/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml new file mode 100644 index 00000000..732d78b3 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..b199f36c --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,109 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + - name: cloud-provider + value: external + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10257" + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "100" + scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10259" + - name: profiling + value: "false" + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 + - name: auto-compaction-mode + value: periodic + - name: auto-compaction-retention + value: 8h + - name: election-timeout + value: "2500" + - name: heartbeat-interval + value: "250" + - name: snapshot-count + value: "6400" + files: + - content: | + --- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + path: /etc/kube-proxy-config.yaml + - content: | + #!/usr/bin/env bash + + # + # (PK) I couldn't find a better/simpler way to conifgure it. See: + # https://github.com/kubernetes-sigs/cluster-api/issues/4512 + # + + set -o errexit + set -o nounset + set -o pipefail + + dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + readonly dir + + # Exit fast if already appended. + if [[ ! -f ${dir}/kube-proxy-config.yaml ]]; then + exit 0 + fi + + # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). + kubeadm_file="/etc/kubeadm.yml" + if [[ ! -f ${kubeadm_file} ]]; then + kubeadm_file="/run/kubeadm/kubeadm.yaml" + fi + + # Run this script only if this is the init node. + if [[ ! -f ${kubeadm_file} ]]; then + exit 0 + fi + + # Append kube-proxy-config.yaml to kubeadm config and delete it + cat "${dir}/kube-proxy-config.yaml" >> "${kubeadm_file}" + rm "${dir}/kube-proxy-config.yaml" + + echo success > /tmp/kube-proxy-patch + owner: root:root + path: /etc/kube-proxy-patch.sh + permissions: "0755" + preKubeadmCommands: + - bash /etc/kube-proxy-patch.sh + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-34/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/scs/1-34/cluster-class/templates/openstack-cluster-template.yaml new file mode 100644 index 00000000..3cc03fbb --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/openstack-cluster-template.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: + identityRef: + cloudName: overridden-by-patch + name: overridden-by-patch + apiServerLoadBalancer: + enabled: false + managedSecurityGroups: + allNodesSecurityGroupRules: + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: VXLAN (Cilium) + portRangeMin: 8472 + portRangeMax: 8472 + protocol: udp + description: "Allow VXLAN traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: HealthCheck (Cilium) + portRangeMin: 4240 + portRangeMax: 4240 + protocol: tcp + description: "Allow HealthCheck traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: Hubble (Cilium) + portRangeMin: 4244 + portRangeMax: 4244 + protocol: tcp + description: "Allow Hubble traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-proxy exporter + portRangeMin: 10249 + portRangeMax: 10249 + protocol: tcp + description: "Allow Prometheus traffic for kube-proxy exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-controller-manager exporter + portRangeMin: 10257 + portRangeMax: 10257 + protocol: tcp + description: "Allow Prometheus traffic for kube-controller-manager exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-scheduler exporter + portRangeMin: 10259 + portRangeMax: 10259 + protocol: tcp + description: "Allow Prometheus traffic for kube-scheduler exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus node exporter + portRangeMin: 9100 + portRangeMax: 9100 + protocol: tcp + description: "Allow Prometheus traffic for scraping node exporter" + controlPlaneNodesSecurityGroupRules: + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus etcd exporter + portRangeMin: 2381 + portRangeMax: 2381 + protocol: tcp + description: "Allow Prometheus traffic for scraping etcd exporter" \ No newline at end of file diff --git a/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-control-plane.yaml b/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-control-plane.yaml new file mode 100644 index 00000000..703c1b1c --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-control-plane.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-worker.yaml new file mode 100644 index 00000000..920dbb0a --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/templates/openstack-machine-template-worker.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-34/cluster-class/values.yaml b/providers/openstack/scs/1-34/cluster-class/values.yaml new file mode 100644 index 00000000..1344cf85 --- /dev/null +++ b/providers/openstack/scs/1-34/cluster-class/values.yaml @@ -0,0 +1,50 @@ +# ClusterClass variable defaults +# These are referenced by the ClusterClass template and can be overridden per deployment. +# Variables apply to all nodes by default. Use topology.controlPlane.variables.overrides +# or topology.workers.machineDeployments[].variables.overrides to differentiate. +variables: + # Image configuration + imageName: "ubuntu-capi-image" + imageIsOrc: false + imageAddVersion: true + + # API server + disableAPIServerFloatingIP: false + apiServerLoadBalancer: "octavia-ovn" + certSANs: [] + + # Network + dnsNameservers: ["9.9.9.9", "149.112.112.112"] + nodeCIDR: "10.8.0.0/20" + + # Machine configuration (override per CP/worker via topology) + flavor: "SCS-2V-4" + rootDisk: 50 + serverGroupID: "" + additionalBlockDevices: [] + + # Access management + sshKeyName: "" + securityGroups: [] + securityGroupIDs: [] + + # Cluster-level (control plane only) + controlPlaneAvailabilityZones: [] + controlPlaneOmitAvailabilityZone: false + + # Identity + identityRef: + name: "openstack" + cloudName: "openstack" + + # Container runtime + registryMirrors: [] + + # OIDC + oidcConfig: + clientID: "" + issuerURL: "" + usernameClaim: "preferred_username" + groupsClaim: "groups" + usernamePrefix: "oidc:" + groupsPrefix: "oidc:" diff --git a/providers/openstack/scs/1-34/clusteraddon.yaml b/providers/openstack/scs/1-34/clusteraddon.yaml new file mode 100644 index 00000000..bea5fc78 --- /dev/null +++ b/providers/openstack/scs/1-34/clusteraddon.yaml @@ -0,0 +1,30 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply diff --git a/providers/openstack/scs/1-34/csctl.yaml b/providers/openstack/scs/1-34/csctl.yaml new file mode 100644 index 00000000..5e8fd276 --- /dev/null +++ b/providers/openstack/scs/1-34/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.34 + provider: + apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 + type: openstack diff --git a/providers/openstack/scs/1-34/stack.yaml b/providers/openstack/scs/1-34/stack.yaml new file mode 100644 index 00000000..45e8730d --- /dev/null +++ b/providers/openstack/scs/1-34/stack.yaml @@ -0,0 +1,6 @@ +provider: openstack +clusterStackName: scs +kubernetesVersion: 1.34.6 +addons: + ccm: 2.34.x + csi: 2.34.x diff --git a/providers/openstack/scs/1-35/cluster-addon/ccm/Chart.yaml b/providers/openstack/scs/1-35/cluster-addon/ccm/Chart.yaml new file mode 100644 index 00000000..1a120416 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/ccm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CCM +name: CCM +version: v1 +dependencies: + - alias: openstack-cloud-controller-manager + name: openstack-cloud-controller-manager + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.35.0 diff --git a/providers/openstack/scs/1-35/cluster-addon/ccm/overwrite.yaml b/providers/openstack/scs/1-35/cluster-addon/ccm/overwrite.yaml new file mode 100644 index 00000000..39076ecd --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/ccm/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + openstack-cloud-controller-manager: + cluster: + name: {{ .Cluster.metadata.name }} diff --git a/providers/openstack/scs/1-35/cluster-addon/ccm/values.yaml b/providers/openstack/scs/1-35/cluster-addon/ccm/values.yaml new file mode 100644 index 00000000..3f290366 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/ccm/values.yaml @@ -0,0 +1,21 @@ +openstack-cloud-controller-manager: + secret: + enabled: true + name: ccm-cloud-config + create: true + nodeSelector: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + extraVolumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + extraVolumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + cloudConfig: + global: + use-clouds: true diff --git a/providers/openstack/scs/1-35/cluster-addon/cni/Chart.yaml b/providers/openstack/scs/1-35/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..95516e98 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/cni/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CNI +name: CNI +version: v1 +dependencies: + - alias: cilium + name: cilium + repository: https://helm.cilium.io/ + version: 1.19.2 diff --git a/providers/openstack/scs/1-35/cluster-addon/cni/overwrite.yaml b/providers/openstack/scs/1-35/cluster-addon/cni/overwrite.yaml new file mode 100644 index 00000000..8f86f6a2 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/cni/overwrite.yaml @@ -0,0 +1,5 @@ +values: | + cilium: + k8sServiceHost: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + k8sServicePort: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + ipv4NativeRoutingCIDR: "{{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 0 }}" diff --git a/providers/openstack/scs/1-35/cluster-addon/cni/values.yaml b/providers/openstack/scs/1-35/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..03fa1a33 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/cni/values.yaml @@ -0,0 +1,32 @@ +cilium: + namespaceOverride: kube-system + tls: + secretsNamespace: + name: "kube-system" + sessionAffinity: true + sctp: + enabled: true + bpf: + masquerade: true + endpointRoutes: + enabled: true + bandwidthManager: + enabled: true + egressGateway: + enabled: true + ipam: + mode: "kubernetes" + kubeProxyReplacement: true + kubeProxyReplacementHealthzBindAddr: "0.0.0.0:10256" + routingMode: native + autoDirectNodeRoutes: true + directRoutingSkipUnreachable: true + localRedirectPolicies: + enabled: true + k8s: + requireIPv4PodCIDR: true + policyDenyResponse: icmp + gatewayAPI: + enabled: true + secretsNamespace: + name: "kube-system" diff --git a/providers/openstack/scs/1-35/cluster-addon/csi/Chart.yaml b/providers/openstack/scs/1-35/cluster-addon/csi/Chart.yaml new file mode 100644 index 00000000..b7d9ee53 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/csi/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: CSI +name: CSI +version: v1 +dependencies: + - alias: openstack-cinder-csi + name: openstack-cinder-csi + repository: https://kubernetes.github.io/cloud-provider-openstack + version: 2.35.0 diff --git a/providers/openstack/scs/1-35/cluster-addon/csi/overwrite.yaml b/providers/openstack/scs/1-35/cluster-addon/csi/overwrite.yaml new file mode 100644 index 00000000..d191a115 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/csi/overwrite.yaml @@ -0,0 +1,3 @@ +values: | + openstack-cinder-csi: + clusterID: "{{ .Cluster.metadata.name }}" diff --git a/providers/openstack/scs/1-35/cluster-addon/csi/values.yaml b/providers/openstack/scs/1-35/cluster-addon/csi/values.yaml new file mode 100644 index 00000000..4e648a4f --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/csi/values.yaml @@ -0,0 +1,41 @@ +openstack-cinder-csi: + secret: + enabled: true + name: csi-cloud-config + create: true + filename: cloud.conf + data: + cloud.conf: |- + [Global] + use-clouds = "true" + clouds-file = /etc/openstack/clouds.yaml + storageClass: + delete: + isDefault: true + csi: + plugin: + volumes: + - name: clouds-yaml + secret: + secretName: clouds-yaml + - name: cloud-conf + secret: + secretName: csi-cloud-config + volumeMounts: + - name: clouds-yaml + readOnly: true + mountPath: /etc/openstack + - name: cloud-conf + readOnly: true + mountPath: /etc/kubernetes + - name: cloud-conf + readOnly: true + mountPath: /etc/config + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/providers/openstack/scs/1-35/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/scs/1-35/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..2ac06b1a --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +type: application +description: Metrics Server +name: metrics-server +version: v1 +dependencies: + - name: "metrics-server" + version: "3.13.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/providers/openstack/scs/1-35/cluster-addon/metrics-server/overwrite.yaml b/providers/openstack/scs/1-35/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..7b1dcd5b --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,4 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" diff --git a/providers/openstack/scs/1-35/cluster-addon/metrics-server/values.yaml b/providers/openstack/scs/1-35/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..a89bf027 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,4 @@ +metrics-server: + fullnameOverride: metrics-server + args: + - --kubelet-insecure-tls diff --git a/providers/openstack/scs/1-35/cluster-class/Chart.yaml b/providers/openstack/scs/1-35/cluster-class/Chart.yaml new file mode 100644 index 00000000..4b02c544 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +description: "This chart installs and configures: + + * Openstack scs Cluster Class + + " +name: openstack-scs-1-35-cluster-class +type: application +version: v1 diff --git a/providers/openstack/scs/1-35/cluster-class/templates/_helpers.tpl b/providers/openstack/scs/1-35/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/providers/openstack/scs/1-35/cluster-class/templates/cluster-class.yaml b/providers/openstack/scs/1-35/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..0f56d662 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,890 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + machineInfrastructure: + templateRef: + kind: OpenStackMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker + variables: + # Image variables + - name: imageName + required: false + schema: + openAPIV3Schema: + type: string + description: | + The base name of the OpenStack image used for provisioning servers. + If `imageIsOrc` is enabled, this name refers to an ORC image resource. + If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project. + If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5) + default: {{ .Values.variables.imageName | quote }} + - name: imageIsOrc + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Indicates whether the image name refers to an ORC image resource. + If set to true (default), the `imageName` is interpreted as a reference to an ORC image. + If set to false, the `imageName` is used to filter images in the OpenStack project instead. + default: {{ .Values.variables.imageIsOrc }} + - name: imageAddVersion + required: false + schema: + openAPIV3Schema: + type: boolean + description: | + Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5. + default: {{ .Values.variables.imageAddVersion }} + # Network variables + - name: networkExternalID + required: false + schema: + openAPIV3Schema: + type: string + example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + format: "uuid4" + description: |- + ID of an external OpenStack network. Required when multiple + external networks exist and VMs need public internet access. + - name: networkMTU + required: false + schema: + openAPIV3Schema: + type: integer + example: 1500 + description: "MTU for the private cluster network. Set this to avoid fragmentation issues." + - name: dnsNameservers + required: false + schema: + openAPIV3Schema: + type: array + description: "DNS nameservers for the cluster subnet. Only used when a new network/subnet is created." + default: {{ .Values.variables.dnsNameservers | toJson }} + example: ["9.9.9.9", "149.112.112.112"] + items: + type: string + - name: nodeCIDR + required: false + schema: + openAPIV3Schema: + type: string + format: "cidr" + default: {{ .Values.variables.nodeCIDR | quote }} + example: "10.8.0.0/20" + description: |- + CIDR for the cluster subnet. CAPO will create a network, subnet, + and router. Leave empty to skip network creation. + # Machine configuration + # These apply to all nodes by default. Use topology.controlPlane.variables.overrides + # or topology.workers.machineDeployments[].variables.overrides to differentiate. + - name: flavor + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.flavor | quote }} + example: "SCS-2V-4" + description: |- + OpenStack instance flavor. Applies to all nodes by default. + Override per control plane or worker via topology variables overrides. + - name: rootDisk + required: false + schema: + openAPIV3Schema: + type: integer + minimum: 0 + example: 50 + default: {{ .Values.variables.rootDisk }} + description: |- + Root disk size in GiB. OpenStack volume will be created and used + instead of an ephemeral disk defined in the flavor. + Set to 0 to use the flavor's ephemeral disk. + - name: serverGroupID + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.serverGroupID | quote }} + example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" + description: "Server group for anti-affinity placement. Override per CP/worker via topology." + - name: additionalBlockDevices + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.additionalBlockDevices | toJson }} + description: |- + Additional block devices (Cinder volumes) to attach to nodes. + Override per CP/worker via topology. + items: + type: object + properties: + name: + type: string + sizeGiB: + type: integer + default: 20 + type: + type: string + default: "__DEFAULT__" + required: ["name"] + # Cluster-level (control plane only, managed by OpenStackClusterTemplate) + - name: controlPlaneAvailabilityZones + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.controlPlaneAvailabilityZones | toJson }} + example: ["nova"] + description: "Availability zones for control plane nodes (OpenStack cluster-level setting)." + items: + type: string + - name: controlPlaneOmitAvailabilityZone + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.controlPlaneOmitAvailabilityZone }} + description: |- + Omit availability zone when creating control plane nodes, letting the + Nova scheduler decide based on other scheduling constraints. + # Access management + - name: sshKeyName + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.sshKeyName | quote }} + example: "capi-keypair" + description: "SSH key to inject into all nodes (for debugging)." + - name: securityGroups + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.securityGroups | toJson }} + example: ["security-group-1"] + description: |- + Extra security groups by name for all nodes. + Ignored if securityGroupIDs is set. Override per CP/worker via topology. + items: + type: string + - name: securityGroupIDs + required: false + schema: + openAPIV3Schema: + format: "uuid4" + type: array + default: {{ .Values.variables.securityGroupIDs | toJson }} + example: ["9ae2f488-30a3-4629-bd51-07acb8eb4278"] + description: |- + Extra security groups by UUID for all nodes. + Takes precedence over securityGroups. Override per CP/worker via topology. + items: + type: string + - name: identityRef + required: false + schema: + openAPIV3Schema: + type: object + default: {{ .Values.variables.identityRef | toJson }} + properties: + name: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.name | quote }} + description: "The name of the secret that carries the OpenStack clouds.yaml" + cloudName: + type: string + example: "openstack" + default: {{ .Values.variables.identityRef.cloudName | quote }} + description: "The name of the cloud to use from the clouds.yaml" + # API server + - name: disableAPIServerFloatingIP + required: false + schema: + openAPIV3Schema: + type: boolean + default: {{ .Values.variables.disableAPIServerFloatingIP }} + description: "Disable the floating IP on the API server load balancer." + - name: certSANs + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.certSANs | toJson }} + example: ["mydomain.example"] + description: "Extra Subject Alternative Names for the API server TLS certificate." + items: + type: string + - name: apiServerLoadBalancer + required: false + schema: + openAPIV3Schema: + type: string + default: {{ .Values.variables.apiServerLoadBalancer | quote }} + example: "octavia-ovn" + description: |- + Load balancer in front of the API server. + Options: none, octavia-amphora, octavia-ovn (default). + - name: apiServerAllowedCIDRs + required: false + schema: + openAPIV3Schema: + type: array + example: ["192.168.10.0/24"] + description: |- + Restrict access to the API server to these CIDRs (network-level ACL). + Requires amphora as load balancer provider (CAPO >= v2.12). + Ensure the management cluster's outgoing IP is included, + otherwise CAPO cannot reconcile the workload cluster. + items: + type: string + - name: oidcConfig + required: false + schema: + openAPIV3Schema: + type: object + properties: + clientID: + type: string + example: "kubectl" + description: "A client id that all tokens must be issued for." + issuerURL: + type: string + example: "https://dex.k8s.scs.community" + description: >- + URL of the provider that allows the API server to + discover public signing keys. Only URLs that use the https:// scheme are + accepted. This is typically the provider's discovery URL, changed to have an + empty path. + usernameClaim: + type: string + example: "preferred_username" + default: {{ .Values.variables.oidcConfig.usernameClaim | quote }} + description: >- + JWT claim to use as the user name. By default sub, + which is expected to be a unique identifier of the end user. Admins can choose + other claims, such as email or name, depending on their provider. However, + claims other than email will be prefixed with the issuer URL to prevent naming + clashes with other plugins. + groupsClaim: + type: string + example: "groups" + default: {{ .Values.variables.oidcConfig.groupsClaim | quote }} + description: "JWT claim to use as the user's group. If the claim is present it must be an array of strings." + usernamePrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.usernamePrefix | quote }} + description: >- + Prefix prepended to username claims to prevent + clashes with existing names (such as system: users). For example, the value + oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and + --oidc-username-claim is a value other than email the prefix defaults to ( + Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value + - can be used to disable all prefixing. + groupsPrefix: + type: string + example: "oidc:" + default: {{ .Values.variables.oidcConfig.groupsPrefix | quote }} + description: >- + Prefix prepended to group claims to prevent clashes + with existing names (such as system: groups). For example, the value oidc: will + create group names like oidc:engineering and oidc:infra. + # Container runtime + - name: registryMirrors + required: false + schema: + openAPIV3Schema: + type: array + default: {{ .Values.variables.registryMirrors | toJson }} + description: "Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror." + items: + type: object + properties: + hostnameUpstream: + type: string + example: "docker.io" + description: "The hostname of the upstream registry." + urlUpstream: + type: string + example: "https://registry-1.docker.io" + description: "The server URL of the upstream registry." + urlMirror: + type: string + example: "https://registry.example.com/v2/dockerhub" + description: "The URL of the mirror registry." + certMirror: + type: string + example: "" + description: "TLS certificate of the mirror in PEM format (optional)." + # + # Patches + # + patches: + # + # Patches for OpenStackClusterTemplate resource. + # + - name: apiServerLoadBalancerOctaviaAmphora + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-amphora." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-amphora" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "amphora" + - name: apiServerLoadBalancerOctaviaOVN + description: "Takes care of the patches that should be applied when variable apiServerLoadBalancer is set to octavia-ovn." + enabledIf: {{ `'{{ eq .apiServerLoadBalancer "octavia-ovn" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/apiServerLoadBalancer/enabled" + value: true + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/provider" + value: "ovn" + - name: apiServerAllowedCIDRs + description: "Restricts API server access to the given CIDRs (requires amphora LB)." + enabledIf: {{ `'{{ and .apiServerAllowedCIDRs (eq .apiServerLoadBalancer "octavia-amphora") }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/apiServerLoadBalancer/allowedCIDRs" + valueFrom: + variable: apiServerAllowedCIDRs + - name: networkExternalID + description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." + enabledIf: {{ `'{{ if .networkExternalID }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/externalNetwork" + value: {} + - op: add + path: "/spec/template/spec/externalNetwork/id" + valueFrom: + variable: networkExternalID + - name: networkMTU + description: "Sets the network MTU when variable networkMTU exist in cluster resource." + enabledIf: {{ `'{{ if .networkMTU }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/networkMTU" + valueFrom: + variable: networkMTU + - name: controlPlaneAvailabilityZones + description: "Sets the availability zones which control plane machines may be deployed to." + enabledIf: {{ `'{{ if .controlPlaneAvailabilityZones }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneAvailabilityZones" + valueFrom: + variable: controlPlaneAvailabilityZones + - name: controlPlaneOmitAvailabilityZone + description: "Causes availability zone to be omitted when creating control plane nodes." + enabledIf: {{ `'{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" + valueFrom: + variable: controlPlaneOmitAvailabilityZone + - name: identityRef + description: "Sets the OpenStack identity reference." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/identityRef + valueFrom: + variable: identityRef + - name: nodeCIDRSubnet + description: |- + Sets the NodeCIDR for the OpenStack Subnet to be created. + Cluster actuator will create a network, a subnet with NodeCIDR, + and a router connected to this subnet. + enabledIf: {{ `'{{ if .nodeCIDR }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSubnets" + valueFrom: + template: | + - cidr: '{{ `{{ .nodeCIDR }}` }}' + dnsNameservers: + {{ `{{- range .dnsNameservers }}` }} + - {{ `{{ . }}` }} + {{ `{{- end }}` }} + - name: disableAPIServerFloatingIP + description: "DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server." + enabledIf: {{ `"{{ if .disableAPIServerFloatingIP }}true{{end}}"` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/disableAPIServerFloatingIP" + valueFrom: + variable: disableAPIServerFloatingIP + - name: podCIDROpenStackCluster + description: "Configures pod-network CIDR based security-group rules for native routing." + enabledIf: {{ `'{{- if .builtin.cluster.network.pods -}}true{{- end -}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/managedSecurityGroups/allNodesSecurityGroupRules/-" + valueFrom: + template: | + remoteIPPrefix: {{ `{{ index .builtin.cluster.network.pods 0 }}` }} + direction: ingress + etherType: IPv4 + name: Pod traffic (Cilium native routing) + description: Allow pod-to-pod traffic for Cilium native routing + # + # Patches for OpenStackMachineTemplate resources (image). + # Image patches must stay separate because they use different builtin variables: + # - .builtin.controlPlane.version for CP + # - .builtin.machineDeployment.version for workers + # + - name: controlPlaneImage + description: "Sets the OpenStack image for control plane nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.controlPlane.version }}{{ end }}` }} + - name: workerImage + description: "Sets the OpenStack image for worker nodes." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/image + valueFrom: + template: | + {{ `{{ if .imageIsOrc }}imageRef{{ else }}filter{{ end }}` }}: + name: {{ `{{ .imageName }}{{ if .imageAddVersion }}-{{ .builtin.machineDeployment.version }}{{ end }}` }} + # + # Unified machine patches — target both CP and workers. + # Users can override per CP/worker via topology.controlPlane.variables.overrides + # and topology.workers.machineDeployments[].variables.overrides. + # + - name: flavor + description: "Sets the OpenStack instance flavor for all nodes." + enabledIf: {{ `'{{ ne .flavor "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: flavor + - name: rootDisk + description: "Sets the root disk size in GiB. 0 means use ephemeral disk from flavor." + enabledIf: {{ `'{{ if .rootDisk }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/rootVolume" + valueFrom: + template: | + sizeGiB: {{ `{{ .rootDisk }}` }} + - name: serverGroupID + description: "Sets the server group for anti-affinity." + enabledIf: {{ `'{{ ne .serverGroupID "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + id: {{ `{{ .serverGroupID }}` }} + - name: additionalBlockDevices + description: "Attaches additional Cinder volumes to nodes." + enabledIf: {{ `'{{ if .additionalBlockDevices }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: /spec/template/spec/additionalBlockDevices + valueFrom: + template: | + {{ `{{- range .additionalBlockDevices }}` }} + - name: {{ `{{ .name }}` }} + sizeGiB: {{ `{{ .sizeGiB }}` }} + storage: + type: Volume + volume: + type: {{ `{{ .type }}` }} + {{ `{{- end }}` }} + - name: podCIDRAllowedAddressPairsControlPlane + description: "Allows pod-network source addresses on control-plane ports for native routing." + enabledIf: {{ `'{{- if .builtin.cluster.network.pods -}}true{{- end -}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/ports/-" + valueFrom: + template: | + allowedAddressPairs: + - ipAddress: {{ `{{ index .builtin.cluster.network.pods 0 }}` }} + - name: podCIDRAllowedAddressPairsWorker + description: "Allows pod-network source addresses on worker ports for native routing." + enabledIf: {{ `'{{- if .builtin.cluster.network.pods -}}true{{- end -}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/ports/-" + valueFrom: + template: | + allowedAddressPairs: + - ipAddress: {{ `{{ index .builtin.cluster.network.pods 0 }}` }} + # + # Access patches — target both CP and workers. + # securityGroupIDs takes precedence over securityGroups (last patch wins). + # + - name: securityGroups + description: "Sets security groups by name for all nodes." + enabledIf: {{ `'{{ if .securityGroups }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroups }} { filter: { name: {{ . }}}}, {{ end }} ]'` }} + - name: securityGroupIDs + description: "Sets security groups by UUID for all nodes. Takes precedence over securityGroups." + enabledIf: {{ `'{{ if .securityGroupIDs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/securityGroups" + valueFrom: + template: {{ `'[ {{ range .securityGroupIDs }} { id: {{ . }} }, {{ end }} ]'` }} + - name: sshKeyName + description: "Sets the SSH key to inject into all nodes." + enabledIf: {{ `'{{ ne .sshKeyName "" }}'` }} + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/sshKeyName" + valueFrom: + variable: sshKeyName + # + - name: certSANs + description: "certSANs sets extra Subject Alternative Names for the API Server signing cert." + enabledIf: {{ `'{{ if .certSANs }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" + valueFrom: + variable: certSANs + - name: oidcConfig + description: "Configure API Server to use external authentication service." + enabledIf: {{ `'{{ if and .oidcConfig .oidcConfig.clientID .oidcConfig.issuerURL }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-client-id + value: {{ `'{{ .oidcConfig.clientID }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-issuer-url + value: {{ `'{{ .oidcConfig.issuerURL }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-claim + value: {{ `'{{ .oidcConfig.usernameClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-claim + value: {{ `'{{ .oidcConfig.groupsClaim }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-username-prefix + value: {{ `'{{ .oidcConfig.usernamePrefix }}'` }} + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/-" + valueFrom: + template: | + name: oidc-groups-prefix + value: {{ `'{{ .oidcConfig.groupsPrefix }}'` }} + # + # Registry mirror patches + # + - name: registryMirrorsControlPlane + description: "Configure registry mirrors on control plane nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} + - name: registryMirrorsWorker + description: "Configure registry mirrors on worker nodes (containerd + CRI-O)." + enabledIf: {{ `'{{ if .registryMirrors }}true{{end}}'` }} + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/files" + valueFrom: + template: | + {{ `{{- range $r := .registryMirrors }}` }} + - content: | + server = "{{ `{{ $r.urlUpstream }}` }}" + [host."{{ `{{ $r.urlMirror }}` }}"] + capabilities = ["pull","resolve"] + override_path = true + owner: root:root + path: /etc/containerd/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/hosts.toml + permissions: "0644" + - content: | + [[registry]] + prefix = "{{ `{{ $r.hostnameUpstream }}` }}" + location = "{{ `{{ $r.hostnameUpstream }}` }}" + [[registry.mirror]] + location = "{{ `{{ $r.urlMirror }}` }}" + owner: root:root + path: /etc/containers/registries.conf.d/50-mirror-{{ `{{ $r.hostnameUpstream }}` }}.conf + permissions: "0644" + {{ `{{- if $r.certMirror }}` }} + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containerd/certs/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + - content: "{{ `{{ $r.certMirror }}` }}" + owner: root:root + path: /etc/containers/certs.d/{{ `{{ $r.hostnameUpstream }}` }}/ca.crt + permissions: "0644" + {{ `{{- end }}` }} + {{ `{{- end }}` }} diff --git a/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml b/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml new file mode 100644 index 00000000..732d78b3 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-config-template-worker-openstack.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..9d8aec51 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,65 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + - name: cloud-provider + value: external + - name: allocate-node-cidrs + value: "true" + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10257" + - name: profiling + value: "false" + - name: terminated-pod-gc-threshold + value: "100" + scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 + - name: secure-port + value: "10259" + - name: profiling + value: "false" + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 + - name: auto-compaction-mode + value: periodic + - name: auto-compaction-retention + value: 8h + - name: election-timeout + value: "2500" + - name: heartbeat-interval + value: "250" + - name: snapshot-count + value: "6400" + initConfiguration: + skipPhases: + - addon/kube-proxy + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + - name: provider-id + value: 'openstack:///{{ `{{ instance_id }}` }}' + name: '{{ `{{ local_hostname }}` }}' diff --git a/providers/openstack/scs/1-35/cluster-class/templates/openstack-cluster-template.yaml b/providers/openstack/scs/1-35/cluster-class/templates/openstack-cluster-template.yaml new file mode 100644 index 00000000..06110054 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/openstack-cluster-template.yaml @@ -0,0 +1,79 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster +spec: + template: + spec: + identityRef: + cloudName: overridden-by-patch + name: overridden-by-patch + apiServerLoadBalancer: + enabled: false + managedSecurityGroups: + allNodesSecurityGroupRules: + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: HealthCheck (Cilium) + portRangeMin: 4240 + portRangeMax: 4240 + protocol: tcp + description: "Allow HealthCheck traffic for Cilium" + - remoteManagedGroups: + - controlplane + - worker + direction: ingress + etherType: IPv4 + name: Hubble (Cilium) + portRangeMin: 4244 + portRangeMax: 4244 + protocol: tcp + description: "Allow Hubble traffic for Cilium" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus node exporter + portRangeMin: 9100 + portRangeMax: 9100 + protocol: tcp + description: "Allow Prometheus traffic for scraping node exporter" + controlPlaneNodesSecurityGroupRules: + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-controller-manager exporter + portRangeMin: 10257 + portRangeMax: 10257 + protocol: tcp + description: "Allow Prometheus traffic for kube-controller-manager exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus kube-scheduler exporter + portRangeMin: 10259 + portRangeMax: 10259 + protocol: tcp + description: "Allow Prometheus traffic for kube-scheduler exporter" + - remoteManagedGroups: + - worker + direction: ingress + etherType: IPv4 + name: Prometheus etcd exporter + portRangeMin: 2381 + portRangeMax: 2381 + protocol: tcp + description: "Allow Prometheus traffic for scraping etcd exporter" + workerNodesSecurityGroupRules: + - remoteManagedGroups: + - controlplane + direction: ingress + etherType: IPv4 + name: Control plane to workers + description: "Allow traffic from control-plane nodes to worker nodes (kube-apiserver to admission webhooks)" diff --git a/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-control-plane.yaml b/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-control-plane.yaml new file mode 100644 index 00000000..fb304712 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-control-plane.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane +spec: + template: + spec: + flavor: overridden-by-patch + ports: [] + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-worker.yaml new file mode 100644 index 00000000..96dd4462 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/templates/openstack-machine-template-worker.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker +spec: + template: + spec: + flavor: overridden-by-patch + ports: [] + image: + imageRef: + name: overridden-by-patch diff --git a/providers/openstack/scs/1-35/cluster-class/values.yaml b/providers/openstack/scs/1-35/cluster-class/values.yaml new file mode 100644 index 00000000..1344cf85 --- /dev/null +++ b/providers/openstack/scs/1-35/cluster-class/values.yaml @@ -0,0 +1,50 @@ +# ClusterClass variable defaults +# These are referenced by the ClusterClass template and can be overridden per deployment. +# Variables apply to all nodes by default. Use topology.controlPlane.variables.overrides +# or topology.workers.machineDeployments[].variables.overrides to differentiate. +variables: + # Image configuration + imageName: "ubuntu-capi-image" + imageIsOrc: false + imageAddVersion: true + + # API server + disableAPIServerFloatingIP: false + apiServerLoadBalancer: "octavia-ovn" + certSANs: [] + + # Network + dnsNameservers: ["9.9.9.9", "149.112.112.112"] + nodeCIDR: "10.8.0.0/20" + + # Machine configuration (override per CP/worker via topology) + flavor: "SCS-2V-4" + rootDisk: 50 + serverGroupID: "" + additionalBlockDevices: [] + + # Access management + sshKeyName: "" + securityGroups: [] + securityGroupIDs: [] + + # Cluster-level (control plane only) + controlPlaneAvailabilityZones: [] + controlPlaneOmitAvailabilityZone: false + + # Identity + identityRef: + name: "openstack" + cloudName: "openstack" + + # Container runtime + registryMirrors: [] + + # OIDC + oidcConfig: + clientID: "" + issuerURL: "" + usernameClaim: "preferred_username" + groupsClaim: "groups" + usernamePrefix: "oidc:" + groupsPrefix: "oidc:" diff --git a/providers/openstack/scs/1-35/clusteraddon.yaml b/providers/openstack/scs/1-35/clusteraddon.yaml new file mode 100644 index 00000000..bea5fc78 --- /dev/null +++ b/providers/openstack/scs/1-35/clusteraddon.yaml @@ -0,0 +1,30 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + BeforeClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply + AfterClusterUpgrade: + - name: cni + action: apply + - name: metrics-server + action: apply + - name: csi + action: apply + - name: ccm + action: apply diff --git a/providers/openstack/scs/1-35/csctl.yaml b/providers/openstack/scs/1-35/csctl.yaml new file mode 100644 index 00000000..6c5abf1b --- /dev/null +++ b/providers/openstack/scs/1-35/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.35 + provider: + apiVersion: openstack.csctl.clusterstack.x-k8s.io/v1alpha1 + type: openstack diff --git a/providers/openstack/scs/1-35/stack.yaml b/providers/openstack/scs/1-35/stack.yaml new file mode 100644 index 00000000..0f51ebd8 --- /dev/null +++ b/providers/openstack/scs/1-35/stack.yaml @@ -0,0 +1,6 @@ +provider: openstack +clusterStackName: scs +kubernetesVersion: 1.35.3 +addons: + ccm: 2.35.x + csi: 2.35.x diff --git a/providers/openstack/scs/README.md b/providers/openstack/scs/README.md deleted file mode 100644 index 6533d842..00000000 --- a/providers/openstack/scs/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Cluster Stacks - -## Getting started - -```sh -# Create bootstrap cluster -kind create cluster - -# Init Cluster API -export CLUSTER_TOPOLOGY=true -export EXP_CLUSTER_RESOURCE_SET=true -export EXP_RUNTIME_SDK=true -kubectl apply -f https://github.com/k-orc/openstack-resource-controller/releases/latest/download/install.yaml -clusterctl init --infrastructure openstack - -kubectl -n capi-system rollout status deployment -kubectl -n capo-system rollout status deployment - -``` - -values.yaml - -``` -clusterStackVariables: - ociRepository: registry.scs.community/kaas/cluster-stacks -controllerManager: - rbac: - additionalRules: - - apiGroups: - - "openstack.k-orc.cloud" - resources: - - "images" - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -``` - -``` -# Install CSO and CSPO -helm upgrade -i cso \ --n cso-system \ ---create-namespace \ -oci://registry.scs.community/cluster-stacks/cso \ ---values values.yaml - -kubectl create namespace cluster -``` - -``` -# Add secret using csp-helper chart -helm upgrade -i openstack-secrets -n cluster --create-namespace https://github.com/SovereignCloudStack/openstack-csp-helper/releases/latest/download/openstack-csp-helper.tgz -f -``` - -```sh -cat < /tmp/kubeconfig -kubectl get nodes --kubeconfig /tmp/kubeconfig -``` diff --git a/providers/openstack/scs/cluster-addon/ccm/Chart.lock b/providers/openstack/scs/cluster-addon/ccm/Chart.lock deleted file mode 100644 index ed7ad098..00000000 --- a/providers/openstack/scs/cluster-addon/ccm/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: openstack-cloud-controller-manager - repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.32.0 -digest: sha256:251e0efadca5de3ef39707c196a64a5d5292427faf4163ce8295d416d6e8c355 -generated: "2025-05-28T21:50:59.574735035+02:00" diff --git a/providers/openstack/scs/cluster-addon/ccm/Chart.yaml b/providers/openstack/scs/cluster-addon/ccm/Chart.yaml deleted file mode 100644 index 1b767285..00000000 --- a/providers/openstack/scs/cluster-addon/ccm/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v2 -dependencies: -- alias: openstack-cloud-controller-manager - name: openstack-cloud-controller-manager - repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.32.0 -description: CCM -name: openstack-scs-1-32-cluster-addon -type: application -version: v1 diff --git a/providers/openstack/scs/cluster-addon/ccm/charts/openstack-cloud-controller-manager-2.32.0.tgz b/providers/openstack/scs/cluster-addon/ccm/charts/openstack-cloud-controller-manager-2.32.0.tgz deleted file mode 100644 index 9e97d805..00000000 Binary files a/providers/openstack/scs/cluster-addon/ccm/charts/openstack-cloud-controller-manager-2.32.0.tgz and /dev/null differ diff --git a/providers/openstack/scs/cluster-addon/ccm/values.yaml b/providers/openstack/scs/cluster-addon/ccm/values.yaml deleted file mode 100644 index 770706c7..00000000 --- a/providers/openstack/scs/cluster-addon/ccm/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -openstack-cloud-controller-manager: - secret: - enabled: true - name: cloud-config - create: false - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule diff --git a/providers/openstack/scs/cluster-addon/cni/Chart.lock b/providers/openstack/scs/cluster-addon/cni/Chart.lock deleted file mode 100644 index c6919af3..00000000 --- a/providers/openstack/scs/cluster-addon/cni/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: cilium - repository: https://helm.cilium.io/ - version: 1.17.4 -digest: sha256:bbf6df8f80e3eef4e48b2f767734b19ec0c69aa23afc12f5a2f04587383ef089 -generated: "2025-05-28T21:50:53.229284267+02:00" \ No newline at end of file diff --git a/providers/openstack/scs/cluster-addon/cni/Chart.yaml b/providers/openstack/scs/cluster-addon/cni/Chart.yaml deleted file mode 100644 index 8b8dd2f1..00000000 --- a/providers/openstack/scs/cluster-addon/cni/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v2 -dependencies: -- alias: cilium - name: cilium - repository: https://helm.cilium.io/ - version: 1.17.4 -description: CNI -name: openstack-scs-1-32-cluster-addon -type: application -version: v1 diff --git a/providers/openstack/scs/cluster-addon/cni/charts/cilium-1.17.4.tgz b/providers/openstack/scs/cluster-addon/cni/charts/cilium-1.17.4.tgz deleted file mode 100644 index 06abb163..00000000 Binary files a/providers/openstack/scs/cluster-addon/cni/charts/cilium-1.17.4.tgz and /dev/null differ diff --git a/providers/openstack/scs/cluster-addon/cni/values.yaml b/providers/openstack/scs/cluster-addon/cni/values.yaml deleted file mode 100644 index 195534d0..00000000 --- a/providers/openstack/scs/cluster-addon/cni/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -cilium: - tls: - secretsNamespace: - name: kube-system - sessionAffinity: true - sctp: - enabled: true - ipam: - mode: "kubernetes" diff --git a/providers/openstack/scs/cluster-addon/csi/Chart.lock b/providers/openstack/scs/cluster-addon/csi/Chart.lock deleted file mode 100644 index 033ad7f2..00000000 --- a/providers/openstack/scs/cluster-addon/csi/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: openstack-cinder-csi - repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.32.0 -digest: sha256:86fda0773b17ffa6ae496715f005d132d987a79cbf9a7c16ca5eb96942fb630b -generated: "2025-05-28T21:50:56.213810258+02:00" diff --git a/providers/openstack/scs/cluster-addon/csi/Chart.yaml b/providers/openstack/scs/cluster-addon/csi/Chart.yaml deleted file mode 100644 index 980ef00b..00000000 --- a/providers/openstack/scs/cluster-addon/csi/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v2 -dependencies: -- alias: openstack-cinder-csi - name: openstack-cinder-csi - repository: https://kubernetes.github.io/cloud-provider-openstack - version: 2.32.0 -description: CSI -name: openstack-scs-1-32-cluster-addon -type: application -version: v1 diff --git a/providers/openstack/scs/cluster-addon/csi/charts/openstack-cinder-csi-2.32.0.tgz b/providers/openstack/scs/cluster-addon/csi/charts/openstack-cinder-csi-2.32.0.tgz deleted file mode 100644 index 23e67ece..00000000 Binary files a/providers/openstack/scs/cluster-addon/csi/charts/openstack-cinder-csi-2.32.0.tgz and /dev/null differ diff --git a/providers/openstack/scs/cluster-addon/csi/values.yaml b/providers/openstack/scs/cluster-addon/csi/values.yaml deleted file mode 100644 index 83817b87..00000000 --- a/providers/openstack/scs/cluster-addon/csi/values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -openstack-cinder-csi: - secret: - enabled: true - name: cloud-config - create: false - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - storageClass: - delete: - isDefault: true diff --git a/providers/openstack/scs/cluster-addon/metrics-server/.helmignore b/providers/openstack/scs/cluster-addon/metrics-server/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/openstack/scs/cluster-addon/metrics-server/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/openstack/scs/cluster-addon/metrics-server/Chart.lock b/providers/openstack/scs/cluster-addon/metrics-server/Chart.lock deleted file mode 100644 index 56929d7e..00000000 --- a/providers/openstack/scs/cluster-addon/metrics-server/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: metrics-server - repository: https://kubernetes-sigs.github.io/metrics-server/ - version: 3.12.2 -digest: sha256:b79715342d7c10e97664b5f4d79199044f5da6ef40cca906218cff05ca891122 -generated: "2025-05-28T21:51:03.060237224+02:00" diff --git a/providers/openstack/scs/cluster-addon/metrics-server/Chart.yaml b/providers/openstack/scs/cluster-addon/metrics-server/Chart.yaml deleted file mode 100644 index ad407d13..00000000 --- a/providers/openstack/scs/cluster-addon/metrics-server/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v2 -dependencies: -- alias: metrics-server - name: metrics-server - repository: https://kubernetes-sigs.github.io/metrics-server/ - version: 3.12.2 -description: Metrics Server -name: openstack-scs-1-32-cluster-addon -type: application -version: v1 diff --git a/providers/openstack/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz b/providers/openstack/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz deleted file mode 100644 index 4538e8a1..00000000 Binary files a/providers/openstack/scs/cluster-addon/metrics-server/charts/metrics-server-3.12.2.tgz and /dev/null differ diff --git a/providers/openstack/scs/cluster-class/.helmignore b/providers/openstack/scs/cluster-class/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/openstack/scs/cluster-class/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/openstack/scs/cluster-class/templates/cluster-class.yaml b/providers/openstack/scs/cluster-class/templates/cluster-class.yaml deleted file mode 100644 index feeba43b..00000000 --- a/providers/openstack/scs/cluster-class/templates/cluster-class.yaml +++ /dev/null @@ -1,612 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: ClusterClass -metadata: - name: {{ .Release.Name }}-{{ .Chart.Version }} -spec: - controlPlane: - ref: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane - machineInfrastructure: - ref: - kind: OpenStackMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster - workers: - machineDeployments: - - class: default-worker - template: - bootstrap: - ref: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker - variables: - - name: external_id - required: false - schema: - openAPIV3Schema: - type: string - example: "ebfe5546-f09f-4f42-ab54-094e457d42ec" - format: "uuid4" - description: "ExternalNetworkID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." - - name: controller_flavor - required: false - schema: - openAPIV3Schema: - type: string - default: "SCS-2V-4-20s" - example: "SCS-2V-4-20s" - description: "OpenStack instance flavor for control-plane nodes." - - name: worker_flavor - required: false - schema: - openAPIV3Schema: - type: string - default: "SCS-2V-4" - example: "SCS-2V-4" - description: "OpenStack instance flavor for worker nodes." - - name: controller_root_disk - required: false - schema: - openAPIV3Schema: - type: integer - minimum: 1 - example: 25 - description: "Root disk size in GiB for control-plane nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should only be used for the diskless flavors." - - name: worker_root_disk - required: false - schema: - openAPIV3Schema: - type: integer - minimum: 1 - default: 25 - example: 25 - description: "Root disk size in GiB for worker nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should be used for the diskless flavors." - - name: openstack_security_groups - required: false - schema: - openAPIV3Schema: - type: array - default: [] - example: ["security-group-1"] - description: "The names of the security groups to assign to the instance" - items: - type: string - - name: cloud_name - required: false - schema: - openAPIV3Schema: - type: string - default: "openstack" - example: "openstack" - description: "The name of the cloud to use from the clouds secret" - - name: secret_name - required: false - schema: - openAPIV3Schema: - type: string - default: "openstack" - example: "openstack" - description: "The name of the clouds secret" - - name: controller_server_group_id - required: false - schema: - openAPIV3Schema: - type: string - default: "" - example: "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" - description: "The server group to assign the control plane nodes to." - - name: worker_server_group_id - required: false - schema: - openAPIV3Schema: - type: string - default: "" - example: "869fe071-1e56-46a9-9166-47c9f228e297" - description: "The server group to assign the worker nodes to." - - name: ssh_key - required: false - schema: - openAPIV3Schema: - type: string - default: "" - example: "capi-keypair" - description: "The ssh key to inject in the nodes." - - name: apiserver_loadbalancer - required: false - schema: - openAPIV3Schema: - type: string - default: "octavia-amphora" - example: "none, octavia-amphora, octavia-ovn" - description: | - "In this cluster-stack we have two kind of loadbalancers. Each of them has its own configuration variable. This setting here is to configure the loadbalancer that is placed in front of the apiserver. - You can choose from 2 options: - - none: - No loadbalancer solution will be deployed - - octavia-amphora: - (default) Uses openstack's loadbalancer service (provider:amphora) - - octavia-ovn: - Uses openstack's loadbalancer service (provider:ovn) - - name: dns_nameservers - required: false - schema: - openAPIV3Schema: - type: array - description: | - "DNSNameservers is the list of nameservers for the OpenStack Subnet - being created. Set this value when you need to create a new network/subnet - while the access through DNS is required." - default: ["5.1.66.255", "185.150.99.255"] - example: ["5.1.66.255", "185.150.99.255"] - items: - type: string - - name: node_cidr - required: false - schema: - openAPIV3Schema: - type: string - format: "cidr" - default: "10.8.0.0/20" - example: "10.8.0.0/20" - description: | - "NodeCIDR is the OpenStack Subnet to be created. Cluster actuator - will create a network, a subnet with NodeCIDR, and a router - connected to this subnet. If you leave this empty, no network will be created." - - name: certSANs - required: false - schema: - openAPIV3Schema: - type: array - default: [] - example: ["mydomain.example"] - description: "CertSANs sets extra Subject Alternative Names for the API Server signing cert." - items: - type: string - - name: oidc_config - required: false - schema: - openAPIV3Schema: - type: object - properties: - client_id: - type: string - example: "kubectl" - description: "A client id that all tokens must be issued for." - issuer_url: - type: string - example: "https://dex.k8s.scs.community" - description: "URL of the provider that allows the API server to -dis cover public signing keys. Only URLs that use the https:// scheme are -acc epted. This is typically the provider's discovery URL, changed to have an -emp ty path" - username_claim: - type: string - example: "preferred_username" - default: "preferred_username" - description: "JWT claim to use as the user name. By default sub, -whi ch is expected to be a unique identifier of the end user. Admins can choose -oth er claims, such as email or name, depending on their provider. However, -cla ims other than email will be prefixed with the issuer URL to prevent naming -cla shes with other plugins." - groups_claim: - type: string - example: "groups" - default: "groups" - description: "JWT claim to use as the user's group. If the claim -is present it must be an array of strings." - username_prefix: - type: string - example: "oidc:" - default: "oidc:" - description: "Prefix prepended to username claims to prevent -cla shes with existing names (such as system: users). For example, the value -oid c: will create usernames like oidc:jane.doe. If this flag isn't provided and ---o idc-username-claim is a value other than email the prefix defaults to ( -Iss uer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value -- c an be used to disable all prefixing." - groups_prefix: - type: string - example: "oidc:" - default: "oidc:" - description: "Prefix prepended to group claims to prevent clashes -wit h existing names (such as system: groups). For example, the value oidc: will -cre ate group names like oidc:engineering and oidc:infra." - - name: network_mtu - required: false - schema: - openAPIV3Schema: - type: integer - example: 1500 - description: "NetworkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID." - - name: controlPlaneAvailabilityZones - required: false - schema: - openAPIV3Schema: - type: array - example: ["nova"] - description: "ControlPlaneAvailabilityZones is the set of availability zones which control plane machines may be deployed to." - items: - type: string - - name: controlPlaneOmitAvailabilityZone - required: false - schema: - openAPIV3Schema: - type: boolean - example: true - description: "ControlPlaneOmitAvailabilityZone causes availability zone to be omitted when creating control plane nodes, allowing the Nova scheduler to make a decision on which availability zone to use based on other scheduling constraints." - patches: - - name: k8s_version - description: "Sets the openstack node image for workers and the controlplane to the cluster-api image with the version mentioned in spec.topology.version." - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: replace - path: "/spec/template/spec/image/filter/name" - valueFrom: - template: ubuntu-capi-image-{{ `{{ .builtin.cluster.topology.version }}` }} - - name: apiserver_loadbalancer_octavia-amphora - description: "Takes care of the patches that should be applied when variable apiserver_loadbalancer is set to octavia-amphora." - enabledIf: {{ `'{{ eq .apiserver_loadbalancer "octavia-amphora" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: replace - path: "/spec/template/spec/apiServerLoadBalancer/enabled" - value: true - - op: add - path: "/spec/template/spec/apiServerLoadBalancer/provider" - value: "amphora" - - name: apiserver_loadbalancer_octavia-ovn - description: "Takes care of the patches that should be applied when variable apiserver_loadbalancer is set to octavia-ovn." - enabledIf: {{ `'{{ eq .apiserver_loadbalancer "octavia-ovn" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: replace - path: "/spec/template/spec/apiServerLoadBalancer/enabled" - value: true - - op: add - path: "/spec/template/spec/apiServerLoadBalancer/provider" - value: "ovn" - - name: controller_flavor - description: "Sets the openstack instance flavor for the KubeadmControlPlane." - enabledIf: {{ `'{{ ne .controller_flavor "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: replace - path: "/spec/template/spec/flavor" - valueFrom: - variable: controller_flavor - - name: worker_flavor - description: "Sets the openstack instance flavor for the worker nodes." - enabledIf: {{ `'{{ ne .worker_flavor "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: false - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: replace - path: "/spec/template/spec/flavor" - valueFrom: - variable: worker_flavor - - name: controller_root_disk - description: "Sets the root disk size in GiB for control-plane nodes." - enabledIf: {{ `"{{ if .controller_root_disk }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/rootVolume" - valueFrom: - template: | - sizeGiB: {{"{{"}} .controller_root_disk {{"}}"}} - - name: worker_root_disk - description: "Sets the root disk size in GiB for worker nodes." - enabledIf: {{ `"{{ if .worker_root_disk }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: false - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/rootVolume" - valueFrom: - template: | - sizeGiB: {{"{{"}} .worker_root_disk {{"}}"}} - - name: external_id - description: "Sets the ID of an external OpenStack Network. This is necessary to get public internet to the VMs." - enabledIf: {{ `"{{ if .external_id }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/externalNetwork" - value: {} - - op: add - path: "/spec/template/spec/externalNetwork/id" - valueFrom: - variable: external_id - - name: network_mtu - description: "Sets the network MTU when variable network_mtu exist in cluster resource." - enabledIf: {{ `"{{ if .network_mtu }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/networkMTU" - valueFrom: - variable: network_mtu - - name: controlPlaneAvailabilityZones - description: "Sets the availability zones which control plane machines may be deployed to." - enabledIf: {{ `"{{ if .controlPlaneAvailabilityZones }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/controlPlaneAvailabilityZones" - valueFrom: - variable: controlPlaneAvailabilityZones - - name: controlPlaneOmitAvailabilityZone - description: "Causes availability zone to be omitted when creating control plane nodes." - enabledIf: {{ `"{{ if .controlPlaneOmitAvailabilityZone }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/controlPlaneOmitAvailabilityZone" - valueFrom: - variable: controlPlaneOmitAvailabilityZone - - name: openstack_security_groups - description: "Sets the list of the openstack security groups for the worker and the controlplane instances." - enabledIf: {{ `"{{ if .openstack_security_groups }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/securityGroups" - valueFrom: - template: {{ `"[ {{ range .openstack_security_groups }} { filter: { name: {{ . }}}}, {{ end }} ]"` }} - - name: cloud_name - description: "Sets the name of the cloud to use from the clouds secret." - enabledIf: {{ `'{{ ne .cloud_name "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: replace - path: "/spec/template/spec/identityRef/cloudName" - valueFrom: - variable: cloud_name - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: replace - path: "/spec/template/spec/identityRef/cloudName" - valueFrom: - variable: cloud_name - - name: secret_name - description: "Sets the name of the clouds secret." - enabledIf: {{ `'{{ ne .secret_name "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: replace - path: "/spec/template/spec/identityRef/name" - valueFrom: - variable: secret_name - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: replace - path: "/spec/template/spec/identityRef/name" - valueFrom: - variable: secret_name - - name: controller_server_group_id - description: "Sets the server group to assign the control plane nodes to." - enabledIf: {{ `'{{ ne .controller_server_group_id "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/serverGroup" - valueFrom: - template: | - id: {{"{{"}} .controller_server_group_id {{"}}"}} - - name: worker_server_group_id - description: "Sets the server group to assign the worker nodes to." - enabledIf: {{ `'{{ ne .worker_server_group_id "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: false - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/serverGroup" - valueFrom: - template: | - id: {{"{{"}} .worker_server_group_id {{"}}"}} - - name: ssh_key - description: "Sets the ssh key to inject in the nodes." - enabledIf: {{ `'{{ ne .ssh_key "" }}'` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackMachineTemplate - matchResources: - controlPlane: true - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/sshKeyName" - valueFrom: - variable: ssh_key - - name: certSANs - description: "CertSANs sets extra Subject Alternative Names for the API Server signing cert." - enabledIf: {{ `"{{ if .certSANs }}true{{end}}"` }} - definitions: - - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs" - valueFrom: - variable: certSANs - - name: oidc_config - description: "Configure API Server to use external authentication service." - enabledIf: {{ `"{{ if and .oidc_config .oidc_config.client_id .oidc_config.issuer_url }}true{{end}}"` }} - definitions: - - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-client-id" - valueFrom: - variable: oidc_config.client_id - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-issuer-url" - valueFrom: - variable: oidc_config.issuer_url - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-claim" - valueFrom: - variable: oidc_config.username_claim - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-claim" - valueFrom: - variable: oidc_config.groups_claim - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-username-prefix" - valueFrom: - variable: oidc_config.username_prefix - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/oidc-groups-prefix" - valueFrom: - variable: oidc_config.groups_prefix - - name: subnet - description: "Sets the NodeCIDR for the OpenStack Subnet to be created. Cluster actuator will create a network, a subnet with NodeCIDR, and a router connected to this subnet." - enabledIf: {{ `"{{ if .node_cidr }}true{{end}}"` }} - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: OpenStackClusterTemplate - matchResources: - infrastructureCluster: true - jsonPatches: - - op: add - path: "/spec/template/spec/managedSubnets" - valueFrom: - template: | - - cidr: '{{"{{"}} .node_cidr {{"}}"}}' - dnsNameservers: - {{`{{- range .dns_nameservers }}`}} - - {{`{{ . }}`}} - {{`{{- end }}`}} diff --git a/providers/openstack/scs/cluster-class/templates/image.yaml b/providers/openstack/scs/cluster-class/templates/image.yaml deleted file mode 100644 index 2850e378..00000000 --- a/providers/openstack/scs/cluster-class/templates/image.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: openstack.k-orc.cloud/v1alpha1 -kind: Image -metadata: - name: '{{ .Values.images.worker.name }}' -spec: - cloudCredentialsRef: - cloudName: '{{ .Values.identityRef.cloudName }}' - secretName: '{{ .Values.identityRef.name }}' - managementPolicy: managed - resource: - content: - diskFormat: qcow2 - download: - hash: - algorithm: sha256 - value: 8bfad5cb331480a7530952d8d6ff41155c28b2941f8be7a3f0318ebd1569cf83 - url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v1.32/ubuntu-2204-kube-v1.32.5.qcow2 - properties: - architecture: x86_64 - hardware: - diskBus: scsi - qemuGuestAgent: true - rngModel: virtio - scsiModel: virtio-scsi - vifModel: virtio - minDiskGB: 20 - minMemoryMB: 2048 - operatingSystem: - distro: ubuntu - version: '22.04' diff --git a/providers/openstack/scs/cluster-class/templates/openstack-machine-template-control-plane.yaml b/providers/openstack/scs/cluster-class/templates/openstack-machine-template-control-plane.yaml deleted file mode 100644 index d2acdd7d..00000000 --- a/providers/openstack/scs/cluster-class/templates/openstack-machine-template-control-plane.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: OpenStackMachineTemplate -metadata: - name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane -spec: - template: - spec: - flavor: {{ .Values.controller_flavor }} - identityRef: - cloudName: {{ .Values.identityRef.cloudName }} - name: {{ .Values.identityRef.name }} - image: - filter: - name: {{ .Values.images.controlPlane.name }} diff --git a/providers/openstack/scs/cluster-class/templates/openstack-machine-template-worker.yaml b/providers/openstack/scs/cluster-class/templates/openstack-machine-template-worker.yaml deleted file mode 100644 index 83deb117..00000000 --- a/providers/openstack/scs/cluster-class/templates/openstack-machine-template-worker.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: OpenStackMachineTemplate -metadata: - name: {{ .Release.Name }}-{{ .Chart.Version }}-default-worker -spec: - template: - spec: - flavor: {{ .Values.worker_flavor }} - identityRef: - cloudName: {{ .Values.identityRef.cloudName }} - name: {{ .Values.identityRef.name }} - image: - filter: - name: {{ .Values.images.worker.name }} diff --git a/providers/openstack/scs/cluster-class/values.yaml b/providers/openstack/scs/cluster-class/values.yaml deleted file mode 100644 index afb2139f..00000000 --- a/providers/openstack/scs/cluster-class/values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -controller_flavor: SCS-2V-4-20s -identityRef: - cloudName: openstack - name: openstack -images: - controlPlane: - name: ubuntu-capi-image-v1.32.5 - worker: - name: ubuntu-capi-image-v1.32.5 -openstack_loadbalancer_apiserver: false -restrict_kubeapi: [] -worker_flavor: SCS-2V-4-20 diff --git a/providers/openstack/scs/image-manager.yaml b/providers/openstack/scs/image-manager.yaml new file mode 100644 index 00000000..869d4082 --- /dev/null +++ b/providers/openstack/scs/image-manager.yaml @@ -0,0 +1,39 @@ +--- +images: + - name: ubuntu-capi-image + enable: true + format: raw + login: ubuntu + min_disk: 20 + min_ram: 1024 + status: active + visibility: private + multi: false + separator: "-" + meta: + architecture: x86_64 + hw_disk_bus: virtio + hw_rng_model: virtio + hw_scsi_model: virtio-scsi + hw_watchdog_action: reset + hypervisor_type: qemu + os_distro: ubuntu + os_purpose: k8snode + replace_frequency: never + uuid_validity: none + provided_until: none + tags: + - clusterstacks + versions: + - version: 'v1.32.13' + url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2204-kube-v1.32/ubuntu-2204-kube-v1.32.13.qcow2 + checksum: "sha256:bd1c5664e2bc34cc37a424391a44511fd12becb4c2c51e9f6b102e90d992d5fa" + - version: 'v1.33.10' + url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.33/ubuntu-2404-kube-v1.33.10.qcow2 + checksum: "sha256:efc3817b565c407710724b9d7b51cbb433638aad6b613b64843da28d58a777aa" + - version: 'v1.34.6' + url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.34/ubuntu-2404-kube-v1.34.6.qcow2 + checksum: "sha256:c59fb893be8320d7112473290358320fad2756e3e31dce4f90ae8bda9d289a3d" + - version: 'v1.35.3' + url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.35/ubuntu-2404-kube-v1.35.3.qcow2 + checksum: "sha256:2892b11dcac515f8f2fcb6a1b60f04a45f7d7ec8fdb2a3f3ccc255deb72d69b4" diff --git a/providers/openstack/scs/versions.yaml b/providers/openstack/scs/versions.yaml deleted file mode 100644 index 99fc8c73..00000000 --- a/providers/openstack/scs/versions.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- kubernetes: 1.30.13 - cinder_csi: 2.30.3 - occm: 2.30.5 -- kubernetes: 1.31.9 - cinder_csi: 2.31.7 - occm: 2.31.3 -- kubernetes: 1.32.5 - cinder_csi: 2.32.0 - occm: 2.32.0 diff --git a/providers/openstack/scs2/README.md b/providers/openstack/scs2/README.md deleted file mode 100644 index 69b00f9e..00000000 --- a/providers/openstack/scs2/README.md +++ /dev/null @@ -1,149 +0,0 @@ -# Cluster Stacks - -## Prerequisites - -- kind -- kubectl -- helm -- clusterctl (v1.10x) - -## Getting started - - -### Prepare a management Cluster using kind - -```sh -# Create bootstrap cluster -kind create cluster - -# Init Cluster API -export CLUSTER_TOPOLOGY=true -export EXP_CLUSTER_RESOURCE_SET=true -export EXP_RUNTIME_SDK=true -kubectl apply -f https://github.com/k-orc/openstack-resource-controller/releases/latest/download/install.yaml -clusterctl init --infrastructure openstack:v0.12.6 - -kubectl -n capi-system rollout status deployment -kubectl -n capo-system rollout status deployment -``` - -### Install the Cluster Stack Operator - -``` -helm upgrade -i cso \ --n cso-system \ ---create-namespace \ -oci://registry.scs.community/cluster-stacks/cso -``` - -### Prepare some environment variables for reuse - -```sh -export CLUSTER_NAMESPACE=cluster -export CLUSTER_NAME=my-cluster -export CLUSTERSTACK_NAMESPACE=cluster -export CLUSTERSTACK_VERSION=v1 -export OS_CLIENT_CONFIG_FILE=${PWD}/clouds.yaml -kubectl create namespace $CLUSTER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - -``` - -### Add clouds.yaml as Secret - -```sh -# Create secret for CAPO -kubectl create secret -n $CLUSTER_NAMESPACE generic openstack --from-file=clouds.yaml=$OS_CLIENT_CONFIG_FILE --dry-run=client -oyaml | kubectl apply -f - - -# Prepare the Secret as it will be deployed in the Workload Cluster -kubectl create secret -n kube-system generic clouds-yaml --from-file=clouds.yaml=$OS_CLIENT_CONFIG_FILE --dry-run=client -oyaml > clouds-yaml-secret - -# Add the Secret to the ClusterResourceSet Secret in the Management Cluster -kubectl create -n $CLUSTER_NAMESPACE secret generic clouds-yaml --from-file=clouds-yaml-secret --type=addons.cluster.x-k8s.io/resource-set --dry-run=client -oyaml | kubectl apply -f - -``` - -```yaml -cat < /tmp/kubeconfig -kubectl get nodes --kubeconfig /tmp/kubeconfig -``` diff --git a/providers/openstack/scs2/cluster-class/.helmignore b/providers/openstack/scs2/cluster-class/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/providers/openstack/scs2/cluster-class/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/providers/openstack/scs2/cluster-class/values.yaml b/providers/openstack/scs2/cluster-class/values.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/providers/openstack/scs2/image.yaml b/providers/openstack/scs2/image.yaml deleted file mode 100644 index 080ae774..00000000 --- a/providers/openstack/scs2/image.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: openstack.k-orc.cloud/v1alpha1 -kind: Image -metadata: - name: "ubuntu-capi-image-v1.33.6" -spec: - cloudCredentialsRef: - cloudName: "openstack" - secretName: "openstack" - managementPolicy: managed - resource: - visibility: private - properties: - hardware: - diskBus: scsi - scsiModel: virtio-scsi - vifModel: virtio - qemuGuestAgent: true - rngModel: virtio - architecture: x86_64 - minDiskGB: 20 - minMemoryMB: 2048 - operatingSystem: - distro: ubuntu - version: "24.04" - content: - diskFormat: qcow2 - download: - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.33/ubuntu-2404-kube-v1.33.6.qcow2 - hash: - algorithm: sha256 - value: ff458b22c33fc08eca9ba6635783e9a409b6f0613f577c4acdec554db7e2f6a7 diff --git a/providers/openstack/scs2/kubernetes.yaml b/providers/openstack/scs2/kubernetes.yaml deleted file mode 100644 index 86b47884..00000000 --- a/providers/openstack/scs2/kubernetes.yaml +++ /dev/null @@ -1,55 +0,0 @@ ---- -images: - - name: ubuntu-capi-image - enable: true - format: raw - login: ubuntu - min_disk: 20 - min_ram: 1024 - status: active - visibility: public - multi: false - separator: "-" - meta: - architecture: x86_64 - hw_disk_bus: virtio - hw_rng_model: virtio - hw_scsi_model: virtio-scsi - hw_watchdog_action: reset - hypervisor_type: qemu - os_distro: ubuntu - os_purpose: k8snode - replace_frequency: never - uuid_validity: none - provided_until: none - tags: - - clusterstacks - versions: - - version: 'v1.33.4' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.33/ubuntu-2404-kube-v1.33.4.qcow2 - checksum: "sha256:1f55111551d5c9948d4e02215be56a712ed818d9000592d4f11d4b6cc4407ade" - build_date: 2025-12-17 - - version: 'v1.33.5' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.33/ubuntu-2404-kube-v1.33.5.qcow2 - checksum: "sha256:bd9efa9cad5d7028306eb26ecdc42a2f84337542a050381766714c4c0c1f7a98" - build_date: 2025-12-17 - - version: 'v1.33.6' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.33/ubuntu-2404-kube-v1.33.6.qcow2 - checksum: "sha256:ff458b22c33fc08eca9ba6635783e9a409b6f0613f577c4acdec554db7e2f6a7" - build_date: 2025-12-17 - - version: 'v1.34.0' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.34/ubuntu-2404-kube-v1.34.0.qcow2 - checksum: "sha256:1321c0978818752619ab994acccf4e2d9b241aa738fc56ed0a46b0ebe21fedfb" - build_date: 2025-12-17 - - version: 'v1.34.1' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.34/ubuntu-2404-kube-v1.34.1.qcow2 - checksum: "sha256:0f2153d01e13693a680010045b1c7f4c511495eff4f8672ea29b475b02b43dc4" - build_date: 2025-12-17 - - version: 'v1.34.2' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.34/ubuntu-2404-kube-v1.34.2.qcow2 - checksum: "sha256:8d37637dc86cd307e50ba2dab2d96aa3c46933552f5607f4663b40c246adb0a8" - build_date: 2025-12-17 - - version: 'v1.34.3' - url: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images/ubuntu-2404-kube-v1.34/ubuntu-2404-kube-v1.34.3.qcow2 - checksum: "sha256:b3c487345dd8ff2eea6ddd3e526d068abc3a59d40a994581f6dfc7be10df427b" - build_date: 2025-12-17 diff --git a/providers/openstack/scs2/versions.yaml b/providers/openstack/scs2/versions.yaml deleted file mode 100644 index 85cff539..00000000 --- a/providers/openstack/scs2/versions.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- kubernetes: 1.32.8 - cinder_csi: 2.32.2 - occm: 2.32.0 -- kubernetes: 1.33.7 - cinder_csi: 2.33.1 - occm: 2.33.1 -- kubernetes: 1.34.3 - cinder_csi: 2.34.1 - occm: 2.34.1