diff --git a/.github/workflows/superlinter.yml b/.github/workflows/superlinter.yml index b5b9d25..fd90053 100644 --- a/.github/workflows/superlinter.yml +++ b/.github/workflows/superlinter.yml @@ -38,6 +38,10 @@ jobs: VALIDATE_JSON_PRETTIER: false VALIDATE_MARKDOWN_PRETTIER: false VALIDATE_PYTHON_PYLINT: false + # ruff (I001) already enforces import sorting with a 120-char line + # length; isort's 88-char limit cannot fit the noqa'd star imports + # needed for the pytest conftest fixtures. + VALIDATE_PYTHON_ISORT: false VALIDATE_SHELL_SHFMT: false VALIDATE_YAML: false VALIDATE_KUBERNETES_KUBECONFORM: false diff --git a/.trivyignore b/.trivyignore index 8a9588c..93792ba 100644 --- a/.trivyignore +++ b/.trivyignore @@ -3,5 +3,7 @@ AVD-KSV-0020 # Container 'apache' of Deployment 'hello-world' should set 'securi AVD-KSV-0021 # Container 'apache' of Deployment 'hello-world' should set 'securityContext.runAsGroup' > 10000. Not needed on OCP AVD-KSV-0014 # Readonly root filesystem does not work with httpd ubi images AVD-KSV-0125 # Container apache in deployment hello-world (namespace: default) uses an image from an untrusted registry. registry.access.redhat.com is trusted +AVD-KSV-0113 # The k-orc credentials Role must read RHOSO config secrets and manage the ORC cloud-config secret in the openstack namespace; the service account is dedicated to the credentials Job +AVD-KSV-0013 # The pattern utility container is only published as :latest; the credentials Job is short-lived and CI-only diff --git a/tests/ci/conftest.py b/tests/ci/conftest.py index 726fd1f..57dfce7 100644 --- a/tests/ci/conftest.py +++ b/tests/ci/conftest.py @@ -1 +1,9 @@ +import sys +from pathlib import Path + from validatedpatterns_tests.interop.conftest_openshift import * + +# The k-orc feature tree uses a hyphenated directory name (matching the +# upstream project name), which is not importable as a Python package. +# Make its test module importable from tests/ci/test_standalone.py. +sys.path.insert(0, str(Path(__file__).parent / "k-orc")) diff --git a/tests/ci/k-orc/README.md b/tests/ci/k-orc/README.md new file mode 100644 index 0000000..e9b4889 --- /dev/null +++ b/tests/ci/k-orc/README.md @@ -0,0 +1,152 @@ +# k-orc: OpenStack Resource Controller integration + +This directory deploys [ORC][orc] (OpenStack Resource Controller) on the +cluster and manages a small steady-state smoke stack in OpenStack through it: + +- a **Flavor** (`m1.xsmall`: 1024 MiB RAM, 2 vCPUs, 15 GiB disk), +- a CirrOS **Image** (downloaded by ORC from download.cirros-cloud.net, + pinned by sha256), +- a **Network**, **Subnet**, and **Port**, +- a **Server** (VM) booting the managed image with the managed flavor on the + managed port. + +The goal is twofold: exercise ORC end-to-end against a real RHOSO cloud, and +give the pattern CI meaningful assertions (see `test_k_orc.py`). + +This is a CI-only feature: nothing outside `tests/ci/` is modified, and the +standard pattern chart does not deploy it. + +[orc]: https://github.com/k-orc/openstack-resource-controller + +## Layout + +```text +k-orc/ + applications/ # Argo CD Applications, applied by the CI tests + k-orc-crds.yaml # static: orc-crds chart (CRDs), wave -2 + k-orc-operator.yaml # static: orc chart (controller), wave -1 + k-orc-resources.yaml # template: git source filled in by the tests + resources/ # Source path of k-orc-resources (flat directory) + 00-credentials-rbac.yaml # SA/Role/RoleBinding for the credentials Job + 01-credentials-job.yaml # assembles the ORC credential Secret + 10-flavor.yaml ... 15-server.yaml # ORC custom resources (wave 0) + test_k_orc.py # CI assertions (applies the Applications, then polls) +``` + +## How it is deployed + +The CI tests (not the pattern chart) create the Argo CD Applications, in the +namespace of the Argo CD instance managing the pattern: + +1. `test_k_orc.py` reads every manifest in `applications/` and applies it + (create or idempotent patch). Static manifests (`k-orc-crds`, + `k-orc-operator`: OCI Helm chart sources) are applied as-is. The + generated one (`k-orc-resources`) gets its Git source filled in: + `VP_KORC_REPOURL` / `VP_KORC_TARGET_REVISION` environment variables, + then the local Git checkout (origin remote and current branch), then the + deployed parent Application's source. +2. `k-orc-crds` and `k-orc-operator` sync the official ORC Helm charts from + quay.io into `orc-system` (CRDs first, then the controller). +3. `k-orc-resources` syncs `resources/` into the `openstack` namespace. + Until the ORC CRDs are established, Argo CD rejects its sync plan + ("one or more synchronization tasks are not valid"); the Application's + sync retries (limit 20, exponential backoff) make it converge once the + CRDs are in place. Within that Application, sync waves order the + credentials RBAC (-2), the credentials Job (-1) and the ORC custom + resources (0). + +The Applications and everything they deploy are steady-state: they are left +running after the CI run, and re-applying is idempotent. To remove the whole +feature, delete the three Applications (their finalizer cascade removes the +CRDs, the controller, the ORC resources and, through ORC, the OpenStack +resources themselves). + +Applications carrying the `validatedpatterns.io/ci-feature: k-orc` label are +excluded from the pattern inventory assertions in `test_standalone.py`, +since they are created by the CI rather than by the pattern chart. + +## Version pinning + +Both Helm charts (`orc-crds` and `orc`) are pinned to the same version in +`applications/`. Bump both together. `3.0.0-pre1` was the only published +chart tag at the time this was written; prefer a stable release once one is +available. + +## OpenStack credentials + +ORC resources reference `spec.cloudCredentialsRef` pointing at the +`orc-cloud-config` Secret in the `openstack` namespace. That Secret is +assembled by the `k-orc-credentials` Job from RHOSO's own configuration: + +- `openstack-config` ConfigMap: `clouds.yaml` (cloud `default`, without the + password), +- `openstack-config-secret` Secret: `cloudrc` (carries `OS_PASSWORD`), +- `combined-ca-bundle` Secret: the TLS CA bundle (exposed to ORC as the + optional `cacert` key). + +The password is only ever handled in memory inside the Job; it is never +printed, logged, or committed. If the RHOSO admin password is rotated, +delete the completed `k-orc-credentials` Job object: the next Argo sync +re-runs it and refreshes the Secret. The same applies after editing the Job +manifest itself (Jobs are immutable). + +The credential account is the RHOSO admin (`clouds.yaml` entry `default`). +Creating flavors requires elevated OpenStack permissions; if the deployment +uses a restricted account, the `k-orc-flavor` object will report a terminal +`InvalidConfiguration`/`UnrecoverableError` condition. + +## Configuration knobs + +- **Subnet CIDR**: `192.168.150.0/24` by default, in + `resources/13-subnet.yaml`. Change it there if it overlaps existing + networks. `cidr` is immutable on the Subnet object: after changing it, + delete the `k-orc-subnet`, `k-orc-port` and `k-orc-server` objects and let + Argo re-create them. +- **Image**: pinned to CirrOS 0.6.3 (URL + sha256) in + `resources/11-image.yaml`. Bump both together using the published + `SHA256SUMS`. +- **Flavor**: `m1.xsmall`, 1024 MiB / 2 vCPUs / 15 GiB in + `resources/10-flavor.yaml`. + +## Pruning + +Pruning is intentionally disabled (`Prune=false`) on all three Applications: +GitOps changes to the manifests do not delete OpenStack resources, and +neither CRDs nor the controller are removed on manifest removal. Deleting +the Applications is the supported way to remove everything (see above). + +## Namespaces + +- `orc-system`: ORC controller (created by the chart applications). +- `openstack`: ORC custom resources, the credentials Job and its Secret. +- The Argo CD namespace hosting the Applications is discovered at runtime + from the deployed pattern Applications; the manifests' hardcoded + `namespace: vp-gitops` is only a documented default that the tests + override. + +## Running the tests + +`test_k_orc.py` is collected through `tests/ci/test_standalone.py` (the +`k-orc` directory name is not a valid Python package name, so +`tests/ci/conftest.py` puts it on `sys.path` and `test_standalone.py` +star-imports it). Run as usual: + +```bash +VP_HUBCONFIG= ./pattern.sh make run-ci-tests +``` + +Environment knobs: `VP_KORC_TEST_TIMEOUT` (default 900s), +`VP_KORC_POLL_INTERVAL` (default 5s), `VP_KORC_REPOURL` / +`VP_KORC_TARGET_REVISION` (override the k-orc-resources source). The first +deployment can take a few minutes (chart pulls, controller start, image +download, VM build), hence the larger default timeout compared to the rest +of the suite. + +The tests apply the Applications and then assert: + +- `k-orc-crds`, `k-orc-operator` and `k-orc-resources` applications are + Synced and Healthy, +- the ORC CRDs are installed and the controller Deployment is Available, +- every smoke stack object is `Available=True` / `Progressing=False` with a + populated `status.id`, failing fast on terminal error reasons, +- the server boots from the managed image and the port has a fixed IP. diff --git a/tests/ci/k-orc/applications/k-orc-crds.yaml b/tests/ci/k-orc/applications/k-orc-crds.yaml new file mode 100644 index 0000000..6905a74 --- /dev/null +++ b/tests/ci/k-orc/applications/k-orc-crds.yaml @@ -0,0 +1,48 @@ +--- +# Installs the ORC Custom Resource Definitions from the official ORC Helm +# chart published on quay.io. +# +# This Application is created by the "k-orc" application rendered by the +# rhoso-gitops chart (see charts/all/rhoso-gitops/values.yaml). It must run +# before the controller (k-orc-operator.yaml) and before any ORC custom +# resources are applied. +# +# Pruning is intentionally disabled: CRDs are cluster-scoped and deleting +# them would delete every ORC custom resource in the cluster. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: k-orc-crds + # The namespace must match the Argo CD instance watching Applications. + namespace: vp-gitops + labels: + # Identifies applications owned by CI features. The pattern inventory + # test excludes applications with this label. + validatedpatterns.io/ci-feature: k-orc + annotations: + argocd.argoproj.io/sync-wave: "-2" + finalizers: + - resources-finalizer.argocd.argoproj.io/background +spec: + project: default + source: + # Helm OCI chart: / is split between repoURL/chart. + repoURL: quay.io + chart: orc/helm/orc-crds + # Keep in sync with k-orc-operator.yaml. Both charts must use the same + # released version. + targetRevision: 3.0.0-pre1 + destination: + server: https://kubernetes.default.svc + namespace: orc-system + syncPolicy: + automated: {} + retry: + limit: 20 + backoff: + duration: 30s + factor: 2 + maxDuration: 5m + syncOptions: + - Prune=false + - CreateNamespace=true diff --git a/tests/ci/k-orc/applications/k-orc-operator.yaml b/tests/ci/k-orc/applications/k-orc-operator.yaml new file mode 100644 index 0000000..53e9654 --- /dev/null +++ b/tests/ci/k-orc/applications/k-orc-operator.yaml @@ -0,0 +1,57 @@ +--- +# Deploys the ORC controller from the official ORC Helm chart published on +# quay.io. +# +# This Application is created by the "k-orc" application rendered by the +# rhoso-gitops chart (see charts/all/rhoso-gitops/values.yaml). It runs after +# the CRDs (k-orc-crds.yaml) and before any ORC custom resources are applied. +# +# Pruning is intentionally disabled: GitOps removal of manifests must not +# delete the controller or its resources. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: k-orc-operator + # The namespace must match the Argo CD instance watching Applications. + namespace: vp-gitops + labels: + # Identifies applications owned by CI features. The pattern inventory + # test excludes applications with this label. + validatedpatterns.io/ci-feature: k-orc + annotations: + argocd.argoproj.io/sync-wave: "-1" + finalizers: + - resources-finalizer.argocd.argoproj.io/background +spec: + project: default + source: + # Helm OCI chart: / is split between repoURL/chart. + repoURL: quay.io + chart: orc/helm/orc + # Keep in sync with k-orc-crds.yaml. Both charts must use the same + # released version. + targetRevision: 3.0.0-pre1 + helm: + values: | + # Do not use the upstream default priority class: it is reserved for + # system components on OpenShift. Drop the chart's fixed runAsUser/ + # runAsGroup as well: OpenShift restricted-v2 SCCs require UIDs from + # the namespace's allocated range (the pod keeps runAsNonRoot). + priorityClassName: "" + containerSecurityContext: + runAsUser: null + runAsGroup: null + destination: + server: https://kubernetes.default.svc + namespace: orc-system + syncPolicy: + automated: {} + retry: + limit: 20 + backoff: + duration: 30s + factor: 2 + maxDuration: 5m + syncOptions: + - Prune=false + - CreateNamespace=true diff --git a/tests/ci/k-orc/applications/k-orc-resources.yaml b/tests/ci/k-orc/applications/k-orc-resources.yaml new file mode 100644 index 0000000..64e7b21 --- /dev/null +++ b/tests/ci/k-orc/applications/k-orc-resources.yaml @@ -0,0 +1,49 @@ +--- +# Template for the k-orc-resources Argo CD Application, applied by the CI +# tests (tests/ci/k-orc/test_k_orc.py). +# +# The CI tests fill in spec.source.repoURL and spec.source.targetRevision +# (deployment-specific: the pattern's own repository and branch) and create +# this Application in the namespace of the Argo CD instance managing the +# pattern. Its source path (tests/ci/k-orc/resources) holds the credentials +# Job/RBAC and the ORC custom resources. +# +# Until the ORC CRDs are established (k-orc-crds.yaml), Argo CD rejects this +# Application's sync plan ("one or more synchronization tasks are not valid"); +# the sync retries below make it converge once the CRDs are in place. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: k-orc-resources + # Overwritten by the CI tests with the discovered Argo CD namespace. + namespace: vp-gitops + labels: + # Identifies applications owned by CI features. The pattern inventory + # test excludes applications with this label. + validatedpatterns.io/ci-feature: k-orc + finalizers: + - resources-finalizer.argocd.argoproj.io/background +spec: + project: default + source: + # Filled in by the CI tests. + repoURL: "" + path: tests/ci/k-orc/resources + targetRevision: "" + destination: + server: https://kubernetes.default.svc + namespace: openstack + syncPolicy: + automated: {} + retry: + # Until the ORC CRDs are established the sync plan is rejected; short + # retries cover that window, and exhaust quickly enough that a new + # revision (or a re-created Application) re-triggers a fresh sync + # promptly. + limit: 12 + backoff: + duration: 10s + factor: 2 + maxDuration: 1m + syncOptions: + - Prune=false diff --git a/tests/ci/k-orc/resources/00-credentials-rbac.yaml b/tests/ci/k-orc/resources/00-credentials-rbac.yaml new file mode 100644 index 0000000..7ec848b --- /dev/null +++ b/tests/ci/k-orc/resources/00-credentials-rbac.yaml @@ -0,0 +1,58 @@ +--- +# RBAC for the k-orc credentials Job. The Job reads the RHOSO +# openstack-config ConfigMap (clouds.yaml without the password), the +# openstack-config-secret Secret (cloudrc, which carries OS_PASSWORD) and the +# combined-ca-bundle Secret (CA bundle), and writes the assembled ORC +# credential Secret orc-cloud-config. +# +# Applied by the "k-orc" Argo CD application, sync wave -2 (before the +# credentials Job). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k-orc-credentials + namespace: openstack + annotations: + argocd.argoproj.io/sync-wave: "-2" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: k-orc-credentials + namespace: openstack + annotations: + argocd.argoproj.io/sync-wave: "-2" +rules: + - apiGroups: [""] + resources: [configmaps] + resourceNames: [openstack-config] + verbs: [get] + - apiGroups: [""] + resources: [secrets] + resourceNames: [openstack-config-secret, combined-ca-bundle] + verbs: [get] + - apiGroups: [""] + resources: [secrets] + resourceNames: [orc-cloud-config] + verbs: [get, patch, update] + # Create cannot be restricted by resourceName; the service account is + # dedicated to this Job and can only create secrets in this namespace. + - apiGroups: [""] + resources: [secrets] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: k-orc-credentials + namespace: openstack + annotations: + argocd.argoproj.io/sync-wave: "-2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: k-orc-credentials +subjects: + - kind: ServiceAccount + name: k-orc-credentials + namespace: openstack diff --git a/tests/ci/k-orc/resources/01-credentials-job.yaml b/tests/ci/k-orc/resources/01-credentials-job.yaml new file mode 100644 index 0000000..51e42cf --- /dev/null +++ b/tests/ci/k-orc/resources/01-credentials-job.yaml @@ -0,0 +1,118 @@ +--- +# Assembles the ORC credential Secret (orc-cloud-config) for the RHOSO cloud. +# +# ORC requires a Secret with a "clouds.yaml" data key (and optionally +# "cacert"). RHOSO exposes the cloud definition without the password in the +# openstack-config ConfigMap, and the password only in the cloudrc key of the +# openstack-config-secret Secret. This Job merges both in memory and writes +# the result as the orc-cloud-config Secret used by all ORC custom resources. +# +# The password is never printed, logged, or committed. +# +# Notes: +# - The Job is a tracked Argo resource. If this manifest changes, delete the +# completed Job object first (Jobs are immutable), Argo will then re-create +# and re-run it. +# - If the RHOSO admin password is rotated, delete the completed Job so that +# the next sync re-runs it and refreshes the Secret. +# +# Applied by the "k-orc" Argo CD application, sync wave -1 (after the RBAC, +# before the ORC custom resources). +apiVersion: batch/v1 +kind: Job +metadata: + name: k-orc-credentials + namespace: openstack + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + backoffLimit: 4 + template: + metadata: + labels: + app.kubernetes.io/name: k-orc-credentials + spec: + restartPolicy: Never + serviceAccountName: k-orc-credentials + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: create-credentials + # Same image as the pattern utility container: provides oc and yq. + image: quay.io/validatedpatterns/utility-container:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + privileged: false + # No fixed runAsUser: OpenShift restricted-v2 SCCs assign a UID + # from the namespace's allocated range (the pod keeps + # runAsNonRoot). + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + env: + - name: WORK_DIR + value: /work + # Keep tooling state (oc cache, etc.) on the emptyDir. + - name: HOME + value: /work + command: [/bin/bash, -c] + args: + - | + set -euo pipefail + + OC="oc --server=https://kubernetes.default.svc \ + --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + + clouds="$WORK_DIR/clouds.yaml" + cloudrc="$WORK_DIR/cloudrc" + cacert="$WORK_DIR/cacert" + + $OC get -n openstack configmap openstack-config -o jsonpath='{.data.clouds\.yaml}' > "$clouds" + $OC get -n openstack secret openstack-config-secret -o jsonpath='{.data.cloudrc}' | base64 -d > "$cloudrc" + # Secret data is base64-encoded in jsonpath output: decode it. + $OC get -n openstack secret combined-ca-bundle -o jsonpath='{.data.tls-ca-bundle\.pem}' | base64 -d > "$cacert" + + # Extract OS_PASSWORD from the cloudrc shell exports. The value + # is only ever passed through the environment; it is not echoed. + password="$(sed -n 's/^export OS_PASSWORD=//p' "$cloudrc" | tr -d '"'"'"'')" + if [ -z "$password" ]; then + echo "OS_PASSWORD not found in openstack-config-secret cloudrc" >&2 + exit 1 + fi + + # Inject the password into the cloud entry. The cloud name and + # the non-secret auth fields come from the ConfigMap. + cloud_name="$($OC get -n openstack configmap openstack-config -o jsonpath='{.data.OS_CLOUD}' | tr -d '[:space:]')" + if [ -z "$cloud_name" ]; then + cloud_name="default" + fi + PASSWORD="$password" yq -i ".clouds.\"$cloud_name\".auth.password = strenv(PASSWORD)" "$clouds" + + test -s "$cacert" + + # Server-side apply: the plain client-side apply would store + # the full (base64) secret content in the + # last-applied-configuration annotation, which exceeds the + # 256kB annotation size limit for the ~230kB CA bundle. + $OC create secret generic orc-cloud-config -n openstack \ + --from-file=clouds.yaml="$clouds" \ + --from-file=cacert="$cacert" \ + --dry-run=client -o yaml | $OC apply --server-side -f - + + echo "Secret orc-cloud-config ensured (cloud: $cloud_name)" + volumeMounts: + - name: work + mountPath: /work + volumes: + - name: work + emptyDir: {} diff --git a/tests/ci/k-orc/resources/10-flavor.yaml b/tests/ci/k-orc/resources/10-flavor.yaml new file mode 100644 index 0000000..e3e4503 --- /dev/null +++ b/tests/ci/k-orc/resources/10-flavor.yaml @@ -0,0 +1,25 @@ +--- +# Test flavor for the k-orc smoke stack. +# +# NOTE on disk size: the reference CI environment's single EDPM compute +# reports only 9GB of placement disk inventory (with a 0.9 allocation +# ratio), so the root disk is capped at 8GB to fit. If your environment has +# more disk, adjust this value (the flavor disk is immutable on the ORC +# object: delete k-orc-flavor, k-orc-port and k-orc-server and let Argo +# re-create them after changing it). +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Flavor +metadata: + name: k-orc-flavor + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + name: m1.xsmall + description: k-orc CI smoke test flavor + ram: 1024 + vcpus: 2 + disk: 8 diff --git a/tests/ci/k-orc/resources/11-image.yaml b/tests/ci/k-orc/resources/11-image.yaml new file mode 100644 index 0000000..6ae0bb1 --- /dev/null +++ b/tests/ci/k-orc/resources/11-image.yaml @@ -0,0 +1,30 @@ +--- +# CirrOS image for the k-orc smoke stack, downloaded by ORC from the official +# CirrOS download site. The sha256 hash pins the exact content (and makes ORC +# download the image itself instead of using Glance web-download). +# +# When bumping to a newer CirrOS release, update both the URL and the hash +# from https://download.cirros-cloud.net//SHA256SUMS. +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Image +metadata: + name: cirros + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + tags: + - k-orc-ci + properties: + architecture: x86_64 + content: + containerFormat: bare + diskFormat: qcow2 + download: + url: https://download.cirros-cloud.net/0.6.3/cirros-0.6.3-x86_64-disk.img + hash: + algorithm: sha256 + value: 7d6355852aeb6dbcd191bcda7cd74f1536cfe5cbf8a10495a7283a8396e4b75b diff --git a/tests/ci/k-orc/resources/12-network.yaml b/tests/ci/k-orc/resources/12-network.yaml new file mode 100644 index 0000000..8b7d969 --- /dev/null +++ b/tests/ci/k-orc/resources/12-network.yaml @@ -0,0 +1,16 @@ +--- +# Tenant network for the k-orc smoke stack. +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Network +metadata: + name: k-orc-network + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + description: k-orc CI smoke test network + tags: + - k-orc-ci diff --git a/tests/ci/k-orc/resources/13-subnet.yaml b/tests/ci/k-orc/resources/13-subnet.yaml new file mode 100644 index 0000000..a4386a4 --- /dev/null +++ b/tests/ci/k-orc/resources/13-subnet.yaml @@ -0,0 +1,24 @@ +--- +# Subnet for the k-orc smoke stack. +# +# The CIDR is the one knob most likely to need adjusting for a given +# deployment: change it here if 192.168.150.0/24 overlaps with existing +# networks. Note that cidr is immutable on the ORC object: after changing it, +# delete the Subnet (and the objects depending on it: k-orc-port, +# k-orc-server) and let Argo re-create them. +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Subnet +metadata: + name: k-orc-subnet + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + name: k-orc-subnet + description: k-orc CI smoke test subnet + networkRef: k-orc-network + ipVersion: 4 + cidr: 192.168.150.0/24 diff --git a/tests/ci/k-orc/resources/14-port.yaml b/tests/ci/k-orc/resources/14-port.yaml new file mode 100644 index 0000000..f241242 --- /dev/null +++ b/tests/ci/k-orc/resources/14-port.yaml @@ -0,0 +1,22 @@ +--- +# Explicit port for the k-orc smoke stack server. ORC requires ports to be +# created explicitly (no implicit port creation on the server). +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Port +metadata: + name: k-orc-port + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + name: k-orc-port + description: k-orc CI smoke test port + networkRef: k-orc-network + tags: + - k-orc-ci + # No fixed address: OpenStack allocates one from k-orc-subnet. + addresses: + - subnetRef: k-orc-subnet diff --git a/tests/ci/k-orc/resources/15-server.yaml b/tests/ci/k-orc/resources/15-server.yaml new file mode 100644 index 0000000..4adb7a1 --- /dev/null +++ b/tests/ci/k-orc/resources/15-server.yaml @@ -0,0 +1,20 @@ +--- +# Smoke test server: boots the managed CirrOS image with the managed flavor +# on the managed port/network. This VM is steady-state (it stays running in +# the CI environment); success is asserted by the ORC Available condition. +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Server +metadata: + name: k-orc-server + namespace: openstack +spec: + cloudCredentialsRef: + cloudName: default + secretName: orc-cloud-config + managementPolicy: managed + resource: + name: k-orc-server + imageRef: cirros + flavorRef: k-orc-flavor + ports: + - portRef: k-orc-port diff --git a/tests/ci/k-orc/test_k_orc.py b/tests/ci/k-orc/test_k_orc.py new file mode 100644 index 0000000..dc1dbdc --- /dev/null +++ b/tests/ci/k-orc/test_k_orc.py @@ -0,0 +1,505 @@ +"""GitOps setup and assertions for the k-orc (ORC) integration. + +Unlike the rest of the pattern CI (which is purely read-only against an +already-deployed pattern), this module *creates* the k-orc Argo CD +Applications, from the manifests in tests/ci/k-orc/applications/: + +- `k-orc-crds` and `k-orc-operator` are static (OCI Helm chart sources) + and are applied as-is; +- `k-orc-resources` is generated: its git source (the pattern's own + repository, i.e. the repository these tests run from) is filled in at + runtime. + +Everything else is deployed by those Applications through GitOps: the ORC +CRDs and controller from the official Helm charts, then the credentials +Job and the ORC smoke stack (Flavor, Image, Network, Subnet, Port, Server) +from tests/ci/k-orc/resources/. + +The Applications and their resources are steady-state: they are left in +place after the run, and subsequent runs re-apply them idempotently. +Applications carrying the validatedpatterns.io/ci-feature label are +excluded from the pattern inventory assertions in test_standalone.py. + +The tests are collected through tests/ci/test_standalone.py, which +star-imports this module (the "k-orc" directory name is not importable as a +Python package; tests/ci/conftest.py puts it on sys.path). +""" + +from __future__ import annotations + +import os +import subprocess +import time +from collections.abc import Callable +from pathlib import Path + +import pytest +import yaml +from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError + +ORC_API_VERSION = "openstack.k-orc.cloud/v1alpha1" +ARGO_APPLICATION_API_VERSION = "argoproj.io/v1alpha1" + +# Namespaces and object names, mirroring the manifests under tests/ci/. +ORC_CONTROLLER_NAMESPACE = "orc-system" +ORC_RESOURCE_NAMESPACE = "openstack" +ORC_RESOURCES_APP_NAME = "k-orc-resources" + +ORC_EXPECTED_CRDS = ( + "flavors.openstack.k-orc.cloud", + "images.openstack.k-orc.cloud", + "networks.openstack.k-orc.cloud", + "subnets.openstack.k-orc.cloud", + "ports.openstack.k-orc.cloud", + "servers.openstack.k-orc.cloud", +) + +# (kind, object name) pairs mirroring tests/ci/k-orc/deploy/resources/. +ORC_EXPECTED_RESOURCES = ( + ("Flavor", "k-orc-flavor"), + ("Image", "cirros"), + ("Network", "k-orc-network"), + ("Subnet", "k-orc-subnet"), + ("Port", "k-orc-port"), + ("Server", "k-orc-server"), +) + +APPLICATION_ERROR_CONDITIONS = { + "ComparisonError", + "InvalidSpecError", + "SharedResourceWarning", + "SyncError", +} + +ORC_TERMINAL_REASONS = {"InvalidConfiguration", "UnrecoverableError"} + +PATTERN_LABEL = "validatedpatterns.io/pattern" +CI_FEATURE_LABEL = "validatedpatterns.io/ci-feature" + +K_ORC_DIR = Path(__file__).parent +APPLICATIONS_DIR = K_ORC_DIR / "applications" +REPOSITORY_ROOT = K_ORC_DIR.parents[2] + +POLL_INTERVAL = float(os.getenv("VP_KORC_POLL_INTERVAL", "5")) +TEST_TIMEOUT = float(os.getenv("VP_KORC_TEST_TIMEOUT", "900")) +START_TIME = time.monotonic() + + +def load_yaml(path: Path) -> dict: + with path.open(encoding="utf-8") as yaml_file: + return yaml.safe_load(yaml_file) + + +def git_output(*args: str) -> str: + result = subprocess.run( + ["git", *args], + cwd=REPOSITORY_ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def pattern_name() -> str: + values_global = load_yaml(REPOSITORY_ROOT / "values-global.yaml") + return values_global["global"]["pattern"] + + +def k_orc_application_names() -> list: + """Application names from tests/ci/k-orc/applications/*.yaml.""" + names = [] + for path in sorted(APPLICATIONS_DIR.glob("*.yaml")): + for document in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(document, dict) and document.get("kind") == "Application": + names.append(document["metadata"]["name"]) + assert names, f"No Applications found in {APPLICATIONS_DIR}" + return sorted(set(names)) + + +K_ORC_APPLICATIONS = tuple(k_orc_application_names()) + + +def wait_until(check: Callable[[], None], description: str) -> None: + """Poll check() until it stops raising or TEST_TIMEOUT expires.""" + last_error = "condition did not run" + while True: + try: + check() + except ( + AssertionError, + AttributeError, + NotFoundError, + ResourceNotFoundError, + ) as exc: + last_error = f"{type(exc).__name__}: {exc}" + else: + return + + elapsed = time.monotonic() - START_TIME + remaining = TEST_TIMEOUT - elapsed + if remaining <= 0: + pytest.fail( + f"{description} not satisfied within {TEST_TIMEOUT:g}s: {last_error}" + ) + time.sleep(min(POLL_INTERVAL, remaining)) + + +def get_resource(client, api_version: str, kind: str, namespace: str, name: str): + resource = client.resources.get(api_version=api_version, kind=kind) + try: + return resource.get(name=name, namespace=namespace) + except NotFoundError as exc: + raise AssertionError(f"{kind} {namespace}/{name} not found") from exc + + +def argo_namespace(client) -> str: + """Discover the namespace of the Argo CD instance managing the pattern.""" + applications = client.resources.get( + api_version=ARGO_APPLICATION_API_VERSION, + kind="Application", + ) + namespaces = { + application.metadata.namespace + for application in applications.get().items or [] + if getattr(application.metadata, "labels", None) + and application.metadata.labels.get(PATTERN_LABEL) == pattern_name() + } + assert namespaces, "No pattern Applications found on the cluster" + assert ( + len(namespaces) == 1 + ), f"Pattern Applications are spread across namespaces: {sorted(namespaces)}" + return namespaces.pop() + + +def parent_application_source(client, namespace: str) -> tuple[str, str]: + """Return (repoURL, targetRevision) of the deployed pattern repository. + + The parent Application (the pattern's main application, rendered by the + pattern chart) tracks the repository and branch the pattern was deployed + from. + """ + values_global = load_yaml(REPOSITORY_ROOT / "values-global.yaml") + values_standalone = load_yaml(REPOSITORY_ROOT / "values-standalone.yaml") + cluster_group = values_standalone["clusterGroup"] + parent_name = cluster_group["applications"][values_global["global"]["pattern"]][ + "name" + ] + parent = get_resource( + client, + ARGO_APPLICATION_API_VERSION, + "Application", + namespace, + parent_name, + ) + spec = getattr(parent, "spec", None) + source = getattr(spec, "source", None) + assert ( + source is not None + ), f"Application {namespace}/{parent_name} has no spec.source" + return source.repoURL, source.targetRevision + + +def resolve_application_source(client, namespace: str) -> tuple[str, str]: + """Resolve the source for the k-orc Application. + + Precedence: explicit environment overrides, then the local git checkout + (origin remote and current branch: these tests run from the repository + whose k-orc manifests should be deployed), then the deployed parent + Application's source. + """ + repo_url = os.getenv("VP_KORC_REPOURL") + target_revision = os.getenv("VP_KORC_TARGET_REVISION") + if not repo_url: + origin = git_output("remote", "get-url", "origin") + if origin: + repo_url = origin + if not target_revision: + branch = git_output("branch", "--show-current") + if branch: + target_revision = branch + if not repo_url or not target_revision: + parent_url, parent_revision = parent_application_source(client, namespace) + repo_url = repo_url or parent_url + target_revision = target_revision or parent_revision + assert repo_url and target_revision, ( + "Could not resolve the pattern repository source for the k-orc " + "Application (set VP_KORC_REPOURL / VP_KORC_TARGET_REVISION)" + ) + return repo_url, target_revision + + +_APPLIED: dict[str, str | None] = {"namespace": None} + + +def apply_k_orc_applications(client, namespace: str) -> None: + """Create or update the k-orc Applications from applications/. + + Static manifests (OCI chart sources) are applied as-is; generated ones + (empty repoURL/targetRevision, i.e. k-orc-resources) get the pattern + repository source filled in. All are placed in the Argo CD namespace + discovered on the cluster. + """ + repo_url, target_revision = resolve_application_source(client, namespace) + + applications = client.resources.get( + api_version=ARGO_APPLICATION_API_VERSION, + kind="Application", + ) + for path in sorted(APPLICATIONS_DIR.glob("*.yaml")): + for document in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if not isinstance(document, dict) or document.get("kind") != "Application": + continue + name = document["metadata"]["name"] + document["metadata"]["namespace"] = namespace + source = document["spec"]["source"] + if not source.get("repoURL"): + source["repoURL"] = repo_url + if not source.get("targetRevision"): + source["targetRevision"] = target_revision + try: + applications.get(name=name, namespace=namespace) + except NotFoundError: + applications.create(body=document) + else: + # Converge the spec to the manifest (idempotent merge patch; + # CRDs do not accept the client's default strategic merge + # patch). + applications.patch( + body={"spec": document["spec"]}, + name=name, + namespace=namespace, + content_type="application/merge-patch+json", + ) + + +@pytest.fixture() +def k_orc_applications(openshift_dyn_client) -> tuple: + """Apply the k-orc Applications; return (client, namespace). + + The Applications are intentionally left in place after the tests: the + smoke stack is steady-state and re-applying is idempotent. + + The openshift_dyn_client fixture is function-scoped (vp-qe-test-common + v1), so this fixture is function-scoped too and memoizes its work: the + Applications are verified (and re-created if missing) rather than fully + re-resolved on every test. + """ + client = openshift_dyn_client + applications = client.resources.get( + api_version=ARGO_APPLICATION_API_VERSION, + kind="Application", + ) + + if _APPLIED["namespace"]: + namespace = _APPLIED["namespace"] + missing = False + for name in K_ORC_APPLICATIONS: + try: + applications.get(name=name, namespace=namespace) + except NotFoundError: + missing = True + break + if not missing: + return client, namespace + + namespace = argo_namespace(client) + apply_k_orc_applications(client, namespace) + _APPLIED["namespace"] = namespace + return client, namespace + + +def condition_status(resource, condition_type: str) -> tuple[str | None, str, str]: + """Return (status, reason, message) for a condition type.""" + conditions = getattr(getattr(resource, "status", None), "conditions", None) or [] + for condition in conditions: + if getattr(condition, "type", None) == condition_type: + return ( + getattr(condition, "status", None), + getattr(condition, "reason", None) or "", + getattr(condition, "message", None) or "", + ) + return None, "", "" + + +def assert_orc_object_conditions(kind: str, name: str, resource) -> None: + """Assert an ORC object is Available and not Progressing. + + Terminal error reasons fail immediately: they never resolve without spec + changes, so polling would only waste the timeout budget. + """ + context = f"{kind} {ORC_RESOURCE_NAMESPACE}/{name}" + progressing, reason, message = condition_status(resource, "Progressing") + if progressing == "True" and reason in ORC_TERMINAL_REASONS: + pytest.fail(f"{context} is in terminal state: {reason}: {message}") + available, _, available_message = condition_status(resource, "Available") + assert ( + progressing == "False" + ), f"{context} Progressing is {progressing!r} (reason {reason!r}): {message}" + assert ( + available == "True" + ), f"{context} Available is {available!r}: {available_message}" + resource_id = getattr(getattr(resource, "status", None), "id", None) + assert resource_id, f"{context} has no status.id" + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_k_orc_applications(openshift_dyn_client, k_orc_applications): + _, namespace = k_orc_applications + + def check(): + for application_name in K_ORC_APPLICATIONS: + application = get_resource( + openshift_dyn_client, + ARGO_APPLICATION_API_VERSION, + "Application", + namespace, + application_name, + ) + context = f"Application {namespace}/{application_name}" + status = getattr(application, "status", None) + assert ( + getattr(getattr(status, "sync", None), "status", None) == "Synced" + ), f"{context} is not Synced" + assert ( + getattr(getattr(status, "health", None), "status", None) == "Healthy" + ), f"{context} is not Healthy" + failed_conditions = [ + getattr(condition, "type", None) + for condition in getattr(status, "conditions", None) or [] + if getattr(condition, "type", None) in APPLICATION_ERROR_CONDITIONS + ] + assert ( + not failed_conditions + ), f"{context} has persistent error conditions: {failed_conditions}" + + wait_until(check, "k-orc Argo CD applications convergence") + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_orc_crds(openshift_dyn_client, k_orc_applications): + def check(): + crds = openshift_dyn_client.resources.get( + api_version="apiextensions.k8s.io/v1", kind="CustomResourceDefinition" + ) + existing = {item.metadata.name for item in crds.get().items or []} + missing = sorted(set(ORC_EXPECTED_CRDS) - existing) + assert not missing, f"Missing ORC CRDs: {missing}" + + wait_until(check, "ORC CRDs installed") + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_orc_controller_available(openshift_dyn_client, k_orc_applications): + def check(): + deployments = openshift_dyn_client.resources.get( + api_version="apps/v1", kind="Deployment" + ) + orc_deployments = [ + deployment + for deployment in deployments.get(namespace=ORC_CONTROLLER_NAMESPACE).items + or [] + if (getattr(deployment.metadata, "labels", None) or {}).get( + "app.kubernetes.io/name" + ) + == "orc" + ] + assert ( + orc_deployments + ), f"No ORC controller deployment found in {ORC_CONTROLLER_NAMESPACE}" + for deployment in orc_deployments: + context = ( + f"Deployment {ORC_CONTROLLER_NAMESPACE}/{deployment.metadata.name}" + ) + conditions = getattr(deployment.status, "conditions", None) or [] + available = [ + getattr(condition, "status", None) + for condition in conditions + if getattr(condition, "type", None) == "Available" + ] + assert available == ["True"], f"{context} is not Available" + + wait_until(check, "ORC controller availability") + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +@pytest.mark.parametrize("kind,name", ORC_EXPECTED_RESOURCES) +def test_orc_resource(openshift_dyn_client, k_orc_applications, kind: str, name: str): + def check(): + resource = get_resource( + openshift_dyn_client, + ORC_API_VERSION, + kind, + ORC_RESOURCE_NAMESPACE, + name, + ) + assert_orc_object_conditions(kind, name, resource) + + wait_until(check, f"ORC {kind} {name} readiness") + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_orc_server_uses_managed_resources(openshift_dyn_client, k_orc_applications): + """The server must boot from the managed image on the managed network.""" + + def check(): + server = get_resource( + openshift_dyn_client, + ORC_API_VERSION, + "Server", + ORC_RESOURCE_NAMESPACE, + "k-orc-server", + ) + image = get_resource( + openshift_dyn_client, + ORC_API_VERSION, + "Image", + ORC_RESOURCE_NAMESPACE, + "cirros", + ) + port = get_resource( + openshift_dyn_client, + ORC_API_VERSION, + "Port", + ORC_RESOURCE_NAMESPACE, + "k-orc-port", + ) + + server_status = getattr(server, "status", None) + image_id = getattr(getattr(image, "status", None), "id", None) + assert image_id, "Image cirros has no status.id" + server_image_id = getattr( + getattr(server_status, "resource", None), "imageID", None + ) + assert server_image_id == image_id, ( + f"Server k-orc-server boots image {server_image_id!r}, " + f"expected the managed image {image_id!r}" + ) + + port_resource = getattr(getattr(port, "status", None), "resource", None) + fixed_ips = getattr(port_resource, "fixedIPs", None) or [] + assert fixed_ips, "Port k-orc-port has no fixed IP allocated" + + wait_until(check, "ORC server to managed resources attachment") diff --git a/tests/ci/test_standalone.py b/tests/ci/test_standalone.py index 3cdab66..dfe5937 100644 --- a/tests/ci/test_standalone.py +++ b/tests/ci/test_standalone.py @@ -11,6 +11,14 @@ import yaml from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError +# k-orc (ORC) integration tests. The conftest.py in this directory puts the +# hyphenated k-orc directory on sys.path. Star-import pulls in the test +# functions and their module-scoped fixture; helpers redefined below +# (wait_until etc.) are unaffected because the imported functions bind to +# their own module globals. +from test_k_orc import * # noqa: F401, F403, E402, RUF100 +from test_k_orc import CI_FEATURE_LABEL + PATTERN_API_VERSION = "gitops.hybrid-cloud-patterns.io/v1alpha1" APPLICATION_API_VERSION = "argoproj.io/v1alpha1" OPERATOR_CORES_API_VERSION = "operators.coreos.com/v1alpha1" @@ -38,7 +46,7 @@ START_TIME = time.monotonic() -def load_yaml(path: Path) -> dict: +def load_yaml(path: Path) -> dict: # type: ignore[no-redef] with path.open(encoding="utf-8") as yaml_file: return yaml.safe_load(yaml_file) @@ -152,7 +160,9 @@ def assert_field(actual: object, expected: object, context: str) -> None: assert actual == expected, f"{context}: expected {expected!r}, got {actual!r}" -def get_resource(client, api_version: str, kind: str, namespace: str, name: str): +def get_resource( # type: ignore[no-redef] + client, api_version: str, kind: str, namespace: str, name: str +): resource = client.resources.get(api_version=api_version, kind=kind) try: return resource.get(name=name, namespace=namespace) @@ -168,6 +178,12 @@ def list_application_names(client, namespace: str) -> list[str]: return sorted( application.metadata.name for application in applications.get(namespace=namespace).items or [] + # Applications owned by CI features (e.g. k-orc) are created by the + # tests themselves, not by the pattern chart: exclude them from the + # pattern inventory. + if not (getattr(application.metadata, "labels", None) or {}).get( + CI_FEATURE_LABEL + ) ) @@ -216,7 +232,7 @@ def child_application_names(client, namespace: str, parent_name: str) -> list[st return sorted({child["name"] for child in children}) -def condition_status(resource, condition_type: str) -> str | None: +def condition_status(resource, condition_type: str) -> str | None: # type: ignore[no-redef] conditions = getattr(getattr(resource, "status", None), "conditions", None) or [] for condition in conditions: if getattr(condition, "type", None) == condition_type: @@ -224,7 +240,7 @@ def condition_status(resource, condition_type: str) -> str | None: return None -def wait_until(check: Callable[[], None], description: str) -> None: +def wait_until(check: Callable[[], None], description: str) -> None: # type: ignore[no-redef] last_error = "condition did not run" while True: try: @@ -311,14 +327,7 @@ def check(): f"{sorted(deployed_helm_applications)}" ) - applications = openshift_dyn_client.resources.get( - api_version=APPLICATION_API_VERSION, - kind="Application", - ) - actual = sorted( - application.metadata.name - for application in applications.get(namespace=namespace).items or [] - ) + actual = list_application_names(openshift_dyn_client, namespace) assert actual == expected, ( f"Application inventory in {namespace}: expected {expected}, " f"got {actual}; missing {sorted(set(expected) - set(actual))}, "