From 4f3650c2ecec2f3f562851596e139911166dde1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Tue, 3 Nov 2026 20:55:50 +0100 Subject: [PATCH 01/14] Add shadow planning for selective integration tests --- .github/workflows/ci.yml | 62 +++++++++++ docs/ci-test-selection.md | 62 +++++++++++ scripts/ci_test_plan.json | 114 ++++++++++++++++++++ scripts/ci_test_plan.py | 190 ++++++++++++++++++++++++++++++++ scripts/test_ci_test_plan.py | 202 +++++++++++++++++++++++++++++++++++ 5 files changed, 630 insertions(+) create mode 100644 docs/ci-test-selection.md create mode 100644 scripts/ci_test_plan.json create mode 100644 scripts/ci_test_plan.py create mode 100644 scripts/test_ci_test_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b26d2efa0..c2e8b22a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,29 @@ on: - "**" jobs: + # Shadow mode: source decisions never gate the existing test jobs. Only a + # successful main run may become the baseline for a later source comparison. + integration-plan: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Compare integration inputs with passed main evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 scripts/ci_test_plan.py --github-baseline \ + --output "$RUNNER_TEMP/plan.json" --summary "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ci-integration-shadow-plan + path: ${{ runner.temp }}/plan.json + if-no-files-found: error + retention-days: 14 + test-support: runs-on: ubuntu-latest steps: @@ -1099,3 +1122,42 @@ jobs: echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" - name: Verify environment, signals and detached process isolation run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v + + # Publish exactly the source plan whose full integration matrix just passed. + # A skipped, failed or cancelled matrix must never certify those inputs. + integration-evidence: + if: ${{ always() }} + needs: [integration-plan, rspec-compatibility, minitest-compatibility, + pytest-compatibility, javascript-platform-compatibility, jest-compatibility, + vitest-compatibility, mocha-compatibility, cypress-compatibility, + playwright-compatibility, cucumber-compatibility, jest-testdrive-workflow, + testdrive-language-workflow, ruby-testdrive, java-testdrive, + testdrive-vitest-workflow, onboard-apply-workflow, testdrive-mocha-workflow, + playwright-latest, testdrive-playwright-workflow, testdrive-cypress-workflow, + testdrive-cucumber-workflow, testdrive-webdriverio-workflow] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require the complete integration matrix + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json, os + failed = {name: job["result"] for name, job in json.loads(os.environ["RESULTS"]).items() + if job["result"] != "success"} + if failed: + raise SystemExit(f"Integration evidence incomplete: {failed}") + PY + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: github.ref == 'refs/heads/main' + with: + name: ci-integration-shadow-plan + path: ${{ runner.temp }}/integration-evidence + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: github.ref == 'refs/heads/main' + with: + name: ci-integration-source-evidence-v1 + path: ${{ runner.temp }}/integration-evidence/plan.json + if-no-files-found: error + retention-days: 30 diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md new file mode 100644 index 000000000..4bb10de85 --- /dev/null +++ b/docs/ci-test-selection.md @@ -0,0 +1,62 @@ +# Integration test selection + +The CI workflow runs an integration input planner in **shadow mode**. Every +existing test job still runs. The job summary reports which suites have changed +source inputs and which are candidates for reusing earlier evidence. The JSON +artifact, `ci-integration-shadow-plan`, retains every source digest, each suite's +fingerprint and changed paths, and the baseline's run and revision. + +The baseline is an artifact from one of the ten most recent successful `CI` +push runs on `main` in this repository. The workflow publishes that artifact +only after every integration matrix passes. Failed, skipped or cancelled jobs +cannot publish evidence. Missing, expired, incompatible or unavailable baseline +artifacts make every suite run in the plan. + +## Input ownership + +[`scripts/ci_test_plan.json`](../scripts/ci_test_plan.json) declares each suite's +jobs, owned input paths and dependencies on other suites. Shared and +unclassified files affect every suite, including changes to the module files, +CLI entry point, workflow, planner and manifest. Only explicitly listed +documentation paths are ignored; embedded instructions are inputs. +`additional_inputs` adds a consumer without narrowing ownership: generic +onboarding helpers still affect every suite. + +For example, a change to `internal/testdrive/python_validation.go` affects the +Python suite. A JavaScript platform change affects every JavaScript suite and +onboarding. Some Jest-named files implement shared validation and feature +probing; those files conservatively affect every suite. Fixtures without an +explicit owner also affect every suite. + +Content, path and executable/symlink type are fingerprinted. Modification times +do not matter. Added, deleted and renamed inputs invalidate their consumers. +The planner snapshots tracked and non-ignored untracked files, so local previews +include edits without requiring a commit. Keep preview outputs outside the +checkout. + +## Inspect a local plan + +The planner needs Python 3 and Git, with no third-party Python packages: + +```sh +python3 scripts/ci_test_plan.py --output /tmp/plan.json --summary /tmp/plan.md +# Compare a second snapshot with the first one: +python3 scripts/ci_test_plan.py --baseline /tmp/plan.json --output /tmp/next-plan.json +``` + +Use `--github-baseline --repository DataDog/ddtest` instead of `--baseline` to +read passed main evidence with an authenticated GitHub CLI. The artifact records +the original source snapshot, before test setup modifies module files. + +## Before enabling reuse + +`source_unchanged` is not permission to skip a test. Dependencies installed by +the fixtures, floating versions such as `latest`, Go and language toolchains, +runner/browser identities and execution flags must also be resolved and checked. +The plan explicitly records `external_inputs_verified: false` until that work is +implemented. The current artifact certifies source inputs only. + +Review shadow decisions against complete runs, especially edits to shared +helpers, embedded scripts and new fixtures. Keep the full integration matrix +while validating the ownership rules. Any later reuse must preserve required +checks and attach the original passed run's provenance to the current result. diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json new file mode 100644 index 000000000..703190e6b --- /dev/null +++ b/scripts/ci_test_plan.json @@ -0,0 +1,114 @@ +{ + "ignored": ["docs/**", "README.md", "AGENTS.md", "ddto/docs/**", "ddto/README.md"], + "shared": ["internal/testdrive/jest*.go"], + "always_run": [ + "test-support", "test-shards", "test", "test-ddtest-instrument-shards", + "test-ddtest-instrument", "lint", "windows-probe-diagnostics", + "onboarding-evaluator-isolation", "integration-plan", "integration-evidence" + ], + "suites": { + "python": { + "jobs": ["pytest-compatibility", "testdrive-language-workflow"], + "inputs": [ + "internal/compatibility/pytest*", "internal/framework/pytest*", + "internal/platform/python*", "internal/platform/scripts/python_*", + "internal/onboard/python*", "internal/onboard/instructions/pytest.md", + "internal/testdrive/python*", "internal/testdrive/pytest*", + "internal/testdrive/scripts/pytest_*" + ] + }, + "ruby": { + "jobs": ["rspec-compatibility", "minitest-compatibility", "ruby-testdrive"], + "inputs": [ + "internal/compatibility/rspec*", "internal/compatibility/minitest*", + "internal/framework/rspec*", "internal/framework/minitest*", + "internal/platform/ruby*", "internal/platform/scripts/ruby_*", + "internal/onboard/ruby*", "internal/onboard/instructions/ruby*", + "internal/onboard/python_ruby_parity_test.go", + "internal/testdrive/ruby*", "internal/testdrive/scripts/ruby_*", + "internal/testdrive/python_ruby_parity_test.go" + ] + }, + "java": { + "jobs": ["java-testdrive"], + "inputs": [ + "internal/framework/jvm*", "internal/platform/java*", "internal/gradle/**", + "internal/maven/**", "internal/onboard/java*", "internal/onboard/instructions/java*", + "internal/testdrive/java*" + ] + }, + "javascript": { + "jobs": ["javascript-platform-compatibility"], + "inputs": [ + "internal/platform/javascript*", "internal/platform/node_options*", + "internal/platform/scripts/javascript*", "internal/framework/javascript*", + "internal/onboard/javascript*", "internal/onboard/node_*" + ] + }, + "jest": { + "jobs": ["jest-compatibility", "jest-testdrive-workflow"], + "depends_on": ["javascript"], + "inputs": ["internal/compatibility/jest*", "internal/framework/jest*", "internal/onboard/jest*", "internal/testdrive/jest*"] + }, + "vitest": { + "jobs": ["vitest-compatibility", "testdrive-vitest-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/compatibility/vitest*", "internal/framework/vitest*", "internal/framework/scripts/vitest*", + "internal/onboard/vitest*", "internal/onboard/instructions/vitest.md", + "internal/testdrive/vitest*", "internal/testdrive/scripts/vitest*" + ] + }, + "mocha": { + "jobs": ["mocha-compatibility", "testdrive-mocha-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/compatibility/mocha*", "internal/framework/mocha*", "internal/framework/scripts/mocha*", + "internal/onboard/mocha*", "internal/onboard/instructions/mocha.md", + "internal/testdrive/mocha*", "internal/testdrive/scripts/mocha*" + ] + }, + "cypress": { + "jobs": ["cypress-compatibility", "testdrive-cypress-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/compatibility/cypress*", "internal/framework/cypress*", "internal/framework/scripts/cypress*", + "internal/onboard/cypress*", "internal/onboard/instructions/cypress.md", + "internal/testdrive/cypress*", "internal/testdrive/scripts/cypress*" + ] + }, + "playwright": { + "jobs": ["playwright-compatibility", "playwright-latest", "testdrive-playwright-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/compatibility/playwright*", "internal/framework/playwright*", "internal/framework/scripts/playwright*", + "internal/onboard/playwright*", "internal/onboard/instructions/playwright.md", + "internal/testdrive/playwright*", "internal/testdrive/scripts/playwright*" + ] + }, + "cucumber": { + "jobs": ["cucumber-compatibility", "testdrive-cucumber-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/compatibility/cucumber*", "internal/framework/cucumber*", + "internal/onboard/cucumber*", "internal/onboard/instructions/cucumber.md", + "internal/testdrive/cucumber*", "internal/testdrive/scripts/cucumber*" + ] + }, + "webdriverio": { + "jobs": ["testdrive-webdriverio-workflow"], + "depends_on": ["javascript"], + "inputs": [ + "internal/framework/webdriverio*", "internal/framework/scripts/webdriverio*", "internal/platform/webdriverio*", + "internal/onboard/webdriverio*", "internal/onboard/instructions/webdriverio.md", + "internal/testdrive/webdriverio*" + ] + }, + "onboarding": { + "jobs": ["onboard-apply-workflow"], + "depends_on": ["javascript", "vitest", "mocha", "cucumber", "webdriverio"], + "inputs": ["internal/testdrive/apply*"], + "additional_inputs": ["internal/onboard/**"] + } + } +} diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py new file mode 100644 index 000000000..0980b5784 --- /dev/null +++ b/scripts/ci_test_plan.py @@ -0,0 +1,190 @@ +"""Shadow integration selection: compare source inputs with a passed main run. + +This never skips tests. Matching sources still need matching resolved dependencies +and runtime identities before their test results could be reused. +""" + +import argparse +from fnmatch import fnmatchcase +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import tempfile + + +SCHEMA = 1 +ARTIFACT = "ci-integration-source-evidence-v1" +CONFIG = Path(__file__).with_suffix(".json") + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def affected_suites(path, config): + if any(fnmatchcase(path, pattern) for pattern in config["ignored"]): + return set() + suites = config["suites"] + if any(fnmatchcase(path, pattern) for pattern in config["shared"]): + return set(suites) + affected = {name for name, suite in suites.items() + if any(fnmatchcase(path, pattern) for pattern in suite["inputs"])} + if not affected: + return set(suites) # Shared and unclassified inputs invalidate everything. + # Additional consumers cannot turn an otherwise shared path into a private one. + affected.update(name for name, suite in suites.items() + if any(fnmatchcase(path, pattern) for pattern in suite.get("additional_inputs", []))) + while True: + expanded = affected | {name for name, suite in suites.items() + if affected.intersection(suite.get("depends_on", []))} + if expanded == affected: + return affected + affected = expanded + + +def source_inputs(root, config): + result = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], + cwd=root, check=True, capture_output=True) + inputs = {} + for name in sorted(set(os.fsdecode(result.stdout).split("\0")) - {""}): + if not affected_suites(name, config): + continue + path = root / name + try: + mode = path.lstat().st_mode + except FileNotFoundError: + continue # A tracked deletion must disappear from the fingerprint. + if stat.S_ISLNK(mode): + content, kind = os.fsencode(os.readlink(path)), "symlink" + elif stat.S_ISREG(mode): + content, kind = path.read_bytes(), "executable" if mode & 0o111 else "file" + else: + raise ValueError(f"cannot fingerprint {name}: expected a file or symlink") + inputs[name] = [kind, hashlib.sha256(content).hexdigest()] + return inputs + + +def valid_baseline(baseline, repository): + return (isinstance(baseline, dict) and baseline.get("schema") == SCHEMA + and baseline.get("repository") == repository + and isinstance(baseline.get("head"), str) and isinstance(baseline.get("run_id"), str) + and isinstance(baseline.get("inputs"), dict) + and all(isinstance(path, str) and isinstance(value, list) and len(value) == 2 + and value[0] in ("file", "executable", "symlink") + and isinstance(value[1], str) and len(value[1]) == 64 + for path, value in baseline["inputs"].items())) + + +def github_baseline(repository): + """Only completed, successful pushes to main in this workflow are eligible.""" + try: + result = subprocess.run([ + "gh", "run", "list", "--repo", repository, "--workflow", "ci.yml", + "--branch", "main", "--event", "push", "--status", "success", "--limit", "10", + "--json", "databaseId,headSha", + ], check=True, capture_output=True, text=True, timeout=30) + runs = json.loads(result.stdout) + for run in runs: + with tempfile.TemporaryDirectory(prefix="ci-test-baseline-") as directory: + result = subprocess.run([ + "gh", "run", "download", str(run["databaseId"]), "--repo", repository, + "--name", ARTIFACT, "--dir", directory, + ], capture_output=True, timeout=30) + if result.returncode: + continue # Old runs and expired artifacts need no special handling. + try: + baseline = json.loads((Path(directory) / "plan.json").read_text()) + except (OSError, ValueError): + continue + if (valid_baseline(baseline, repository) + and baseline.get("head") == run["headSha"] + and baseline.get("run_id") == str(run["databaseId"])): + return baseline + except (OSError, ValueError, KeyError, subprocess.SubprocessError): + pass # An unavailable baseline can never produce an unchanged-source decision. + return None + + +def make_plan(inputs, config, repository, head, baseline=None): + if not valid_baseline(baseline, repository): + baseline = None + previous = baseline["inputs"] if baseline else {} + changed = sorted(path for path in inputs.keys() | previous.keys() + if inputs.get(path) != previous.get(path)) if baseline else [] + affected = {name: [] for name in config["suites"]} + for path in changed: + for name in affected_suites(path, config): + affected[name].append(path) + suites = {} + for name, suite in config["suites"].items(): + selected = {path: value for path, value in inputs.items() + if name in affected_suites(path, config)} + suites[name] = { + "jobs": suite["jobs"], "fingerprint": digest(selected), + "decision": "source_unchanged" if baseline and not affected[name] else "run", + "reason": ("source inputs unchanged; runtime/dependency verification still required" + if baseline and not affected[name] else + "source inputs changed" if baseline else "no eligible baseline"), + "changed_inputs": affected[name], + } + return { + "schema": SCHEMA, "mode": "shadow", "repository": repository, "head": head, + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "baseline": {key: baseline[key] for key in ("head", "run_id")} if baseline else None, + "external_inputs_verified": False, "always_run": config["always_run"], + "inputs": inputs, "suites": suites, + } + + +def summary(plan): + lines = ["## Integration selection (shadow)", "", + "All tests still run. Unchanged sources are candidates only: resolved dependencies", + "and runtime identities have not been verified. No test evidence is reused.", ""] + if plan["baseline"]: + origin = (f"successful main run {plan['baseline']['run_id']}" + if plan["baseline"]["run_id"] else "local snapshot") + lines.append(f"Compared with {origin} at `{plan['baseline']['head']}`.") + else: + lines.append("No eligible successful main baseline; every suite must run.") + lines += ["", "| Suite | Source decision | Changed inputs |", "| --- | --- | --- |"] + for name, suite in plan["suites"].items(): + paths = suite["changed_inputs"] + details = ", ".join("`" + path.replace("`", "").replace("|", "|") + "`" + for path in paths[:5]) + if len(paths) > 5: + details += f" (+{len(paths) - 5} more)" + lines.append(f"| {name} | {suite['decision']} | {details or '—'} |") + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "local")) + baseline_args = parser.add_mutually_exclusive_group() + baseline_args.add_argument("--baseline", type=Path) + baseline_args.add_argument("--github-baseline", action="store_true") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--summary", type=Path) + args = parser.parse_args() + config = json.loads(CONFIG.read_text()) + baseline = github_baseline(args.repository) if args.github_baseline else None + if args.baseline: + try: + baseline = json.loads(args.baseline.read_text()) + except (OSError, ValueError): + baseline = None + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=args.root, + check=True, capture_output=True, text=True).stdout.strip() + plan = make_plan(source_inputs(args.root, config), config, args.repository, head, baseline) + args.output.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") + if args.summary: + args.summary.write_text(summary(plan)) + print(summary(plan), end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py new file mode 100644 index 000000000..cd5621d6c --- /dev/null +++ b/scripts/test_ci_test_plan.py @@ -0,0 +1,202 @@ +"""Exercise selective CI decisions without using a live GitHub account.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import ci_test_plan as planner + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = json.loads(planner.CONFIG.read_text()) + + +def source(content, kind="file"): + return [kind, hashlib.sha256(content.encode()).hexdigest()] + + +class SelectionTests(unittest.TestCase): + def setUp(self): + self.inputs = {"internal/ext/command.go": source("shared"), + "internal/testdrive/python_validation.go": source("python"), + "internal/testdrive/java_validation.go": source("java")} + self.baseline = planner.make_plan(self.inputs, CONFIG, "DataDog/ddtest", "old") + + def plan(self, inputs, baseline=None): + return planner.make_plan(inputs, CONFIG, "DataDog/ddtest", "new", + self.baseline if baseline is None else baseline) + + def running(self, plan): + return {name for name, suite in plan["suites"].items() if suite["decision"] == "run"} + + def test_same_sources_are_candidates_but_never_certify_external_inputs(self): + plan = self.plan(dict(reversed(list(self.inputs.items())))) + self.assertFalse(self.running(plan)) + self.assertEqual(plan["mode"], "shadow") + self.assertFalse(plan["external_inputs_verified"]) + for name in CONFIG["suites"]: + self.assertEqual(plan["suites"][name]["fingerprint"], + self.baseline["suites"][name]["fingerprint"]) + self.assertIn("All tests still run", planner.summary(plan)) + self.assertIn("runtime identities have not been verified", planner.summary(plan)) + + def test_python_edit_keeps_unrelated_suites_as_candidates(self): + inputs = {**self.inputs, "internal/testdrive/python_validation.go": source("changed")} + plan = self.plan(inputs) + self.assertEqual(self.running(plan), {"python"}) + self.assertEqual(plan["suites"]["python"]["changed_inputs"], + ["internal/testdrive/python_validation.go"]) + + def test_shared_unknown_build_and_workflow_edits_invalidate_every_suite(self): + for path in ("internal/ext/command.go", "internal/testdrive/new_shared.go", + "internal/onboard/ci_shell_state.go", "internal/onboard/new_shared.go", + "go.mod", "go.sum", "main.go", "ddto/cli/cli.go", + ".github/workflows/ci.yml", "scripts/ci_test_plan.json", + "internal/testdrive/jest_features.go", "internal/testdrive/jest_preflight.go", + "internal/testdrive/testdata/new fixture/data.json"): + with self.subTest(path=path): + self.assertEqual(self.running(self.plan({**self.inputs, path: source("changed")})), + set(CONFIG["suites"])) + + def test_test_edits_additions_deletions_renames_and_modes_invalidate_the_owner(self): + old = "internal/testdrive/python_validation.go" + for inputs in ({key: value for key, value in self.inputs.items() if key != old}, + {**self.inputs, old: source("python", "executable")}, + {**self.inputs, "internal/testdrive/python_new_test.go": source("new test")}, + {**{key: value for key, value in self.inputs.items() if key != old}, + "internal/testdrive/java_renamed.go": source("python")}): + with self.subTest(inputs=inputs): + self.assertIn("python", self.running(self.plan(inputs))) + renamed = {**{key: value for key, value in self.inputs.items() if key != old}, + "internal/testdrive/java_renamed.go": source("python")} + self.assertEqual(self.running(self.plan(renamed)), {"python", "java"}) + + def test_embedded_scripts_invalidate_their_consumers(self): + for path, owner in (("internal/testdrive/scripts/pytest_observer.py", "python"), + ("internal/platform/scripts/ruby_env.rb", "ruby"), + ("internal/framework/scripts/vitest.mjs", "vitest"), + ("internal/onboard/instructions/pytest.md", "python")): + with self.subTest(path=path): + self.assertIn(owner, self.running(self.plan({**self.inputs, path: source("changed")}))) + + def test_javascript_inputs_propagate_to_all_javascript_suites_and_onboarding(self): + plan = self.plan({**self.inputs, "internal/platform/scripts/javascript_env.js": source("changed")}) + self.assertEqual(self.running(plan), {"javascript", "jest", "vitest", "mocha", "cypress", + "playwright", "cucumber", "webdriverio", "onboarding"}) + + def test_overlapping_and_transitive_inputs_keep_every_consumer(self): + self.assertEqual(planner.affected_suites("internal/testdrive/python_ruby_parity_test.go", CONFIG), + {"python", "ruby"}) + self.assertEqual(planner.affected_suites("internal/onboard/python_venv.go", CONFIG), + {"python", "onboarding"}) + self.assertEqual(planner.affected_suites("internal/framework/scripts/vitest.mjs", CONFIG), + {"vitest", "onboarding"}) + + def test_only_explicit_documentation_paths_are_ignored(self): + for path in ("docs/onboarding.md", "README.md", "AGENTS.md", "ddto/docs/validation.md"): + self.assertFalse(planner.affected_suites(path, CONFIG)) + self.assertTrue(planner.affected_suites("internal/onboard/instructions/pytest.md", CONFIG)) + + def test_missing_foreign_incompatible_or_malformed_baselines_run_everything(self): + invalid = [None, [], {}, {**self.baseline, "repository": "someone/else"}, + {**self.baseline, "schema": 0}, {**self.baseline, "inputs": []}, + {**self.baseline, "inputs": {"go.mod": "malformed"}}] + for baseline in invalid: + with self.subTest(baseline=baseline): + plan = planner.make_plan(self.inputs, CONFIG, "DataDog/ddtest", "new", baseline) + self.assertEqual(self.running(plan), set(CONFIG["suites"])) + self.assertIsNone(plan["baseline"]) + + +class SnapshotTests(unittest.TestCase): + def test_snapshot_uses_content_and_modes_without_following_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + file = root / "go.mod" + file.write_text("first") + (root / "link").symlink_to("missing-outside-source") + output = subprocess.CompletedProcess([], 0, b"go.mod\0link\0deleted.go\0") + with patch.object(planner.subprocess, "run", return_value=output): + first = planner.source_inputs(root, CONFIG) + os.utime(file, (1, 1)) + self.assertEqual(first, planner.source_inputs(root, CONFIG)) + file.write_text("other") + self.assertNotEqual(first, planner.source_inputs(root, CONFIG)) + file.chmod(0o755) + self.assertEqual(planner.source_inputs(root, CONFIG)["go.mod"][0], "executable") + self.assertNotIn("deleted.go", first) + self.assertEqual(first["link"], source("missing-outside-source", "symlink")) + + +class BaselineTests(unittest.TestCase): + def test_lookup_pins_workflow_branch_event_success_and_artifact_identity(self): + baseline = planner.make_plan({}, CONFIG, "DataDog/ddtest", "good") + baseline["run_id"] = "2" + + def run(command, **kwargs): + if command[1:3] == ["run", "list"]: + return subprocess.CompletedProcess(command, 0, json.dumps([ + {"databaseId": 1, "headSha": "old"}, {"databaseId": 2, "headSha": "good"}])) + if command[3] == "1": + return subprocess.CompletedProcess(command, 1) + (Path(command[-1]) / "plan.json").write_text(json.dumps(baseline)) + return subprocess.CompletedProcess(command, 0) + + with patch.object(planner.subprocess, "run", side_effect=run) as invoke: + self.assertEqual(planner.github_baseline("DataDog/ddtest"), baseline) + command = invoke.call_args_list[0].args[0] + for flag, value in (("--workflow", "ci.yml"), ("--branch", "main"), + ("--event", "push"), ("--status", "success"), ("--repo", "DataDog/ddtest")): + self.assertEqual(command[command.index(flag) + 1], value) + self.assertIn(planner.ARTIFACT, invoke.call_args_list[-1].args[0]) + + baseline["head"] = "wrong-revision" + with patch.object(planner.subprocess, "run", side_effect=run): + self.assertIsNone(planner.github_baseline("DataDog/ddtest")) + + def test_network_authentication_and_timeout_failures_produce_no_baseline(self): + for error in (OSError("missing gh"), subprocess.CalledProcessError(1, ["gh"]), + subprocess.TimeoutExpired(["gh"], 30)): + with self.subTest(error=error), patch.object(planner.subprocess, "run", side_effect=error): + self.assertIsNone(planner.github_baseline("DataDog/ddtest")) + + +class WorkflowTests(unittest.TestCase): + def test_every_job_has_one_owner_and_all_suite_jobs_gate_evidence(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text() + jobs = re.findall(r"^ ([\w-]+):$", workflow.split("jobs:\n", 1)[1], re.M) + registered = CONFIG["always_run"] + [job for suite in CONFIG["suites"].values() for job in suite["jobs"]] + self.assertCountEqual(jobs, registered) + evidence = workflow.split(" integration-evidence:\n", 1)[1] + dependencies = re.search(r" needs: \[([^]]+)\]", evidence).group(1) + self.assertCountEqual(re.split(r",\s*", dependencies), + ["integration-plan"] + [job for suite in CONFIG["suites"].values() for job in suite["jobs"]]) + for suite in CONFIG["suites"].values(): + self.assertTrue(set(suite.get("depends_on", [])).issubset(CONFIG["suites"])) + + def test_shadow_decisions_do_not_gate_existing_jobs(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text().split("jobs:\n", 1)[1] + for block in re.split(r"(?=^ [\w-]+:\n)", workflow, flags=re.M): + if block.startswith((" integration-plan:", " integration-evidence:")): + continue + self.assertNotIn("integration-plan", block) + self.assertNotIn("source_unchanged", block) + + def test_failed_skipped_or_cancelled_jobs_cannot_publish_evidence(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text().split(" integration-evidence:\n", 1)[1] + guard = re.search(r"python3 - <<'PY'\n(.*?)\n PY", workflow, re.S).group(1) + code = "\n".join(line[10:] for line in guard.splitlines()) + for status in ("success", "failure", "skipped", "cancelled"): + result = subprocess.run(["python3", "-c", code], capture_output=True, + env={**os.environ, "RESULTS": json.dumps({"matrix": {"result": status}})}) + self.assertEqual(result.returncode == 0, status == "success") + + +if __name__ == "__main__": + unittest.main() From 9e1271a1c293d63486aa7d235a00dc95a396911e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Tue, 3 Nov 2026 22:09:56 +0100 Subject: [PATCH 02/14] Run affected PR checks and gate releases on full compatibility --- .github/workflows/ci-full.yml | 1132 +++++++++++++++++ .github/workflows/ci.yml | 1218 +++---------------- .github/workflows/go-instrumentation.yml | 20 +- .github/workflows/release.yml | 9 +- Makefile | 4 +- ddto/AGENTS.md | 6 +- ddto/docs/validation.md | 33 +- docs/ci-test-selection.md | 147 ++- internal/releaseworkflow/ci_shards_test.go | 2 +- internal/testdrive/preflight_review_test.go | 2 +- internal/testdrive/vitest_review_test.go | 2 +- scripts/ci_cucumber_matrix.py | 116 ++ scripts/ci_smoke.py | 65 + scripts/ci_test_plan.json | 499 +++++++- scripts/ci_test_plan.py | 258 ++-- scripts/test_ci_cucumber_matrix.py | 154 +++ scripts/test_ci_smoke.py | 35 + scripts/test_ci_test_plan.py | 333 +++-- 18 files changed, 2507 insertions(+), 1528 deletions(-) create mode 100644 .github/workflows/ci-full.yml create mode 100644 scripts/ci_cucumber_matrix.py create mode 100644 scripts/ci_smoke.py create mode 100644 scripts/test_ci_cucumber_matrix.py create mode 100644 scripts/test_ci_smoke.py diff --git a/.github/workflows/ci-full.yml b/.github/workflows/ci-full.yml new file mode 100644 index 000000000..07fbe62a6 --- /dev/null +++ b/.github/workflows/ci-full.yml @@ -0,0 +1,1132 @@ +name: Full compatibility + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + fast-validation: + uses: ./.github/workflows/ci.yml + secrets: inherit + + go-instrumentation: + uses: ./.github/workflows/go-instrumentation.yml + with: + compatibility: full + secrets: inherit + + test-support: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + - name: Test onboarding evaluator + run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' + + # install.sh, and with the runner's pwsh install.ps1, against local release files. + - name: Test install scripts + run: python -m unittest discover -s scripts -p 'test_*.py' + + test-shards: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + # This job already has a compiler-specific cache below. Restoring + # setup-go's cache too collides with read-only module files. + cache: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Cache Go modules + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + # ddto is its own module, checked by its own workflow; its benchmark + # fixtures are deliberately compact. + - name: Format check + run: | + unformatted="$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" + if [ -n "$unformatted" ]; then + echo "The following files are not formatted correctly:" + echo "$unformatted" + exit 1 + fi + + - name: Vet + run: go vet ./... + + - name: Select Go test shard + run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" + + - name: Configure Datadog Test Optimization + if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} + uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 + with: + languages: go + # Pin upstream Go 1.27 compatibility fixes until they reach a stable + # Orchestrion release. + go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 + api_key: ${{ secrets.DATADOG_API_KEY }} + site: datadoghq.com + service: ddtest + + - name: Run tests + run: go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... + env: + DD_ENV: ci + DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" + + - name: Retain shard coverage + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: unit-coverage-${{ matrix.shard }} + path: coverage.out + if-no-files-found: error + overwrite: true + retention-days: 1 + + # Keep the existing required check name; a skipped, cancelled or failed + # shard/support job must not turn the aggregate green. + test: + if: ${{ always() }} + needs: [test-shards, test-support] + runs-on: ubuntu-latest + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + steps: + - name: Require every test job + env: + SHARDS_RESULT: ${{ needs.test-shards.result }} + SUPPORT_RESULT: ${{ needs.test-support.result }} + run: | + test "$SHARDS_RESULT" = success + test "$SUPPORT_RESULT" = success + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: unit-coverage-* + path: ${{ runner.temp }}/unit-coverage + - name: Merge complete coverage + run: python3 scripts/go_test_shards.py merge --expected 3 --output coverage.out "$RUNNER_TEMP"/unit-coverage/*/coverage.out + - name: Upload coverage report to Datadog + if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} + uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 # v1.0.5 + with: + api_key: ${{ secrets.DATADOG_API_KEY }} + site: datadoghq.com + files: coverage.out + format: go-coverprofile + version: v5.19.0 + + # Dogfooding: runs the `test` suite with ddtest instrument, built from this + # checkout as a release builds it, instead of Orchestrion. Its results reach + # Test Optimization beside those of `test`, under this job's name. + test-ddtest-instrument-shards: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + # This job already has a compiler-specific cache below. Restoring + # setup-go's cache too collides with read-only module files. + cache: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + # Its own key, so that its instrumented builds don't replace those of `test`. + - name: Cache Go modules + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + - name: Select Go test shard + run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" + + # The release builder, as `make release` runs it for one target: ddtest + # with the runtime sources and tool wrapper that ddtest instrument provides. + - name: Build ddtest instrument from this checkout + run: | + version="dogfood-${GITHUB_SHA::12}" + go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ + -version "v0.0.0-$version" -ldflags "-X github.com/DataDog/ddtest/internal/buildinfo.Version=$version" \ + -target linux/amd64 -o "$RUNNER_TEMP/dist" + mkdir -p "$RUNNER_TEMP/bin" + mv "$RUNNER_TEMP/dist/ddtest-linux-amd64" "$RUNNER_TEMP/bin/ddtest" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + "$RUNNER_TEMP/bin/ddtest" instrument --version + + - name: Run tests + run: ddtest instrument go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... + env: + DD_CIVISIBILITY_ENABLED: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' && 'parent' || 'false' }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_SERVICE: ddtest + DD_ENV: ci + JOB_CHECK_RUN_ID: ${{ job.check_run_id }} + DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" + # `test` uploads this commit's coverage report. + DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED: "false" + + test-ddtest-instrument: + if: ${{ always() }} + needs: test-ddtest-instrument-shards + runs-on: ubuntu-latest + steps: + - name: Require every instrumented shard + env: + SHARDS_RESULT: ${{ needs.test-ddtest-instrument-shards.result }} + run: test "$SHARDS_RESULT" = success + + rspec-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rspec: "3.12.0" + datadog_ci: "1.31.0" + - rspec: "3.13.2" + datadog_ci: "1.37.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Test RSpec adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install rspec --version "${{ matrix.rspec }}" --no-document + DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$' + DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$' + + minitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - minitest: "5.26.0" + datadog_ci: "1.31.0" + - minitest: "6.0.6" + datadog_ci: "1.37.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Test Minitest adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install minitest --version "${{ matrix.minitest }}" --no-document + gem install rake --version "13.2.1" --no-document + DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$' + + pytest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pytest: "8.4.2" + ddtrace: "4.11.0" + - pytest: "9.1.1" + ddtrace: "4.14.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Test pytest adapter + run: | + python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" pytest-cov pytest-xdist + DDTEST_PYTHON_BINARY="$(command -v python)" \ + go test -v ./internal/compatibility -run '^TestPyTest(AdapterIntegration|SuiteModeSelectorBatches)$' + DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ + go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$' + DDTEST_PYTEST_PYTHON="$(command -v python)" \ + go test -v ./internal/testdrive -run '^TestPytest(RealValidation|CoverageExecutesOnlyInsideCopy|LoggingExecutesOnlyInsideCopy|RejectsEditableImportsFromOriginalCheckout|CoverageDataExecutesOnlyInsideCopy|IgnoredConfigurationUsesRoot|ObserverKeepsConcurrentDuplicateExecutions|PreflightReportsSelectedEnvironment)$' + + javascript-platform-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dd_trace: ["5.111.0", "6.15.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test JavaScript platform + run: | + dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" + npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" + DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestJavaScript(Platform|ActionPreload)Integration$' + + jest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jest: ["27.5.1", "28.1.3", "29.4.3", "29.7.0", "30.5.1"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Jest adapter + run: | + jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" + npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" dd-trace@5.86.0 + DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestJest(Adapter|Testdrive)Integration$' + DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestJestCheckOnlyIntegration$' + + vitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.5"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Vitest adapter + run: | + vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" + # npm 10.9.8 crashes while resolving Vitest's peer dependencies. + # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. + npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "@vitest/coverage-v8@${{ matrix.vitest }}" "dd-trace@5.132.0" + DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ + DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' + DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestVitest(Configuration|ValidationArguments|MultiProjectRejectionKeepsConfigurationEvidence|RepeatedRunOptionIsNormalized)Integration$' + + mocha-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Mocha adapter + run: | + mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" + npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" + DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestMocha.*Integration$' + DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestMochaNativeObserverIntegration$' + + cypress-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Cypress adapter + run: | + cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}" + npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}" + DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ + DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ + DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ + go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$' + + playwright-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Playwright adapter + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + run: | + playwright_dir="${RUNNER_TEMP}/playwright-${{ matrix.playwright }}" + npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}" + DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \ + DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$' + + cucumber-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Cucumber adapter + run: | + cucumber_dir="${RUNNER_TEMP}/cucumber-${{ matrix.cucumber }}" + npm install --prefix "${cucumber_dir}" "@cucumber/cucumber@${{ matrix.cucumber }}" + DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ + DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ + DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ + go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$' + DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestCucumberSupportMetadataIntegration$' + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: golangci-lint + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 + with: + version-file: .golangci-lint-version + args: --timeout=5m + + jest-testdrive-workflow: + name: jest-testdrive-workflow (${{ matrix.node }}, ${{ matrix.jest }}, ${{ matrix.group }} ${{ matrix.shard }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Per runtime: TestPublicJestTestdrive over two shards, plus two groups of + # configuration tests of similar length; together they are the full set. + include: + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} + # Unpinned: catches framework releases our pinned versions predate. + - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + - name: Install real Jest and tracer + run: | + npm install --prefix "${RUNNER_TEMP}/jest-testdrive" jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 + npm ls --prefix "${RUNNER_TEMP}/jest-testdrive" --depth=0 + - name: Validate paired outcomes, all feature probes and cleanup + env: + DDTEST_JEST_NODE_MODULES: ${{ runner.temp }}/jest-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(${{ matrix.tests }})$' -count=1 -timeout=12m + + windows-probe-diagnostics: + runs-on: windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - name: Test Windows probe filesystem diagnostics + run: go test ./internal/testdrive -run '^(TestProbePublicationExplainsWindowsFilesystemErrors|TestReportsCanReplaceExistingDestinationTwice)$' -count=1 + + + testdrive-language-workflow: + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + framework: [pytest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - name: Run Python testdrive + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + run: go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives/${{ matrix.framework }}$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: testdrive-${{ matrix.framework }} + path: ${{ runner.temp }}/ddtest-workflow-reports + + ruby-testdrive: + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - name: pinned-minitest-6 + datadog_ci: "1.40.0" + rspec: "3.13.2" + minitest: "6.0.6" + rake: "13.4.2" + - name: pinned-minitest-5 + datadog_ci: "1.40.0" + rspec: "3.13.2" + minitest: "5.27.0" + rake: "13.4.2" + - name: latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: '3.4' + - name: Install Ruby validation dependencies + env: + DATADOG_CI: ${{ matrix.datadog_ci }} + RSPEC: ${{ matrix.rspec }} + MINITEST: ${{ matrix.minitest }} + RAKE: ${{ matrix.rake }} + run: | + install() { if [ -n "$2" ]; then gem install "$1" --version "$2" --no-document; else gem install "$1" --no-document; fi; } + install datadog-ci "$DATADOG_CI" + install rspec "$RSPEC" + install minitest "$MINITEST" + install rake "$RAKE" + gem install activesupport --conservative --no-document + gem list | grep -E '^(datadog-ci|rspec-core|minitest|rake|activesupport) ' + - name: Run Ruby testdrive integration tests + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_RUN_RUBY_INTEGRATION_TEST: '1' + DDTEST_RUN_RUBY_FORK_INTEGRATION_TEST: '1' + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + run: | + go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives$/^(rspec|minitest)$' -count=1 -timeout=20m + go test -v ./internal/testdrive -run '^TestRuby.+Integration$' -skip '^TestRubyValidationIntegration$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ruby-testdrive-${{ matrix.name }} + path: ${{ runner.temp }}/ddtest-workflow-reports + if-no-files-found: ignore + retention-days: 7 + + java-testdrive: + name: java-testdrive (${{ matrix.name }}, ${{ matrix.java }}, ${{ matrix.tracer }}, ${{ matrix.shard }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # Each scenario runs a full paired validation with real Maven or Gradle builds; + # the scenarios are spread over shards (DDTEST_TEST_SHARD). + name: [pinned, latest] + shard: ['1/4', '2/4', '3/4', '4/4'] + include: + - name: pinned + java: '17' + tracer: '1.67.1' + - name: latest + java: '21' + tracer: latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: temurin + java-version: ${{ matrix.java }} + # The Maven fixtures are generated by the test, so the repository has no + # pom.xml for setup-java's cache key; key the cache on the test instead. + - name: Cache Maven dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ matrix.java }}-${{ hashFiles('internal/testdrive/java_validation_integration_test.go') }} + restore-keys: | + ${{ runner.os }}-maven-${{ matrix.java }}- + # The Gradle fixtures run the gradle on PATH, as a CI job with this + # action does; it also caches the Gradle user home. + - uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 + with: + gradle-version: '8.10.2' + - name: Run Java testdrive integration tests + env: + DDTEST_RUN_JAVA_INTEGRATION_TEST: '1' + DDTEST_JAVA_TRACER_VERSION: ${{ matrix.tracer }} + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + DDTEST_TEST_SHARD: ${{ matrix.shard }} + # Every validation starts Maven many times; short-lived Maven JVMs + # start faster without the optimizing compiler. Both runs inherit it. + MAVEN_OPTS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 + run: go test -v ./internal/testdrive -run '^TestJavaValidationIntegration$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: java-testdrive-${{ matrix.name }}-${{ strategy.job-index }} + path: ${{ runner.temp }}/ddtest-workflow-reports + if-no-files-found: ignore + retention-days: 7 + + testdrive-vitest-workflow: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - vitest: '3.2.7' + tracer: '5.132.0' + - vitest: '4.1.11' + tracer: '6.21.0' + - vitest: '5.0.5' + tracer: '6.21.0' + - vitest: 'latest' + tracer: '6.21.0' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve Vitest version + run: | + version=$(npm view "vitest@${{ matrix.vitest }}" version) + echo "Vitest $version" + echo "DDTEST_VITEST_VERSION=$version" >> "$GITHUB_ENV" + - name: Validate Vitest CLI and Node API paths against local intake + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_VITEST_TRACER_VERSION: ${{ matrix.tracer }} + run: go test -v ./internal/testdrive -run '^TestPublicVitestRunnerTestdrive$' -count=1 -timeout=15m + + onboard-apply-workflow: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Validate generated workflow edits with actionlint + run: | + go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/onboard -run '^TestGeneratedSetupWorkflow.*Actionlint$' -count=1 + - name: Install apply fixture dependencies + run: npm install --prefix "$RUNNER_TEMP/apply-testdrive" vitest@4.1.11 mocha@11.7.6 nyc@17.1.0 dd-trace@6.20.1 @cucumber/cucumber@12.2.0 @wdio/cli@9.20.0 @wdio/local-runner@9.20.0 @wdio/mocha-framework@9.20.0 --no-audit --no-fund + - name: Apply setup and validate against local intake + env: + DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_VITEST_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_CHROME_BINARY: /usr/bin/google-chrome + run: DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/testdrive -run '^TestPublic(OnboardApply|MochaApply|WebdriverIOApply|CucumberApply)$' -count=1 -timeout=8m + + testdrive-mocha-workflow: + runs-on: ubuntu-latest + timeout-minutes: 12 + strategy: + fail-fast: false + matrix: + mocha: ['11.7.6', 'latest'] + shard: ['1/2', '2/2'] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Install Mocha testdrive fixture dependencies + run: | + npm install --prefix "$RUNNER_TEMP/mocha-testdrive" "mocha@${{ matrix.mocha }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.21.0 --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/mocha-testdrive" --depth=0 + - name: Validate CLI, programmatic runner, existing failures and coverage gate + env: + DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/mocha-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(TestPublicMochaTestdrive|TestMochaProbeCoverageThresholdControls|TestMochaDelayedCommandUsesSharedTimeout|TestMochaPreflightResolvesSelectedPackage)$' -count=1 -timeout=10m + + playwright-latest: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + steps: + - id: resolve + run: echo "version=$(npm view @playwright/test version)" >> "$GITHUB_OUTPUT" + + testdrive-playwright-workflow: + needs: playwright-latest + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + playwright: ['1.55.1', 'latest'] + shard: ['1/2', '2/2'] + # The official image ships the browsers and their system packages for this + # exact Playwright release, so nothing is downloaded or installed per run. + container: + image: mcr.microsoft.com/playwright:v${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }}-noble + env: + PLAYWRIGHT_VERSION: ${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + # The workspace is owned by the runner user, not the container's root; + # go build reads it with git for VCS stamping. + - name: Trust the workspace for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Isolate Playwright fixture caches + run: echo "npm_config_cache=$RUNNER_TEMP/playwright-testdrive/cache" >> "$GITHUB_ENV" + - name: Install Playwright testdrive fixture dependencies + run: | + npm install --prefix "$RUNNER_TEMP/playwright-testdrive" "@playwright/test@$PLAYWRIGHT_VERSION" dd-trace@6.20.1 --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/playwright-testdrive" --depth=0 + - name: Validate browser execution, existing failures and supervised cleanup + env: + DDTEST_PLAYWRIGHT_NODE_MODULES: ${{ runner.temp }}/playwright-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(TestPublicPlaywrightTestdrive|TestPlaywrightDiscoveryOwnsImportSideEffects|TestPlaywrightDiscoveryKeepsConfiguredReportsOutOfTheCheckout)$' -count=1 -timeout=12m + + testdrive-cypress-workflow: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Scenarios are split by integration_shard_test.go; each runs exactly + # once per Cypress version. + cypress: ['15.1.0', 'latest'] + shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve Cypress version + id: cypress + run: echo "version=$(npm view "cypress@${{ matrix.cypress }}" version | tail -n 1)" >> "$GITHUB_OUTPUT" + # The Cypress binary downloads during npm install; reuse it per release. + - name: Cache Cypress binary + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/cypress-testdrive-cache + key: ${{ runner.os }}-cypress-binary-${{ steps.cypress.outputs.version }} + - name: Install Cypress testdrive fixture dependencies + env: + CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache + run: | + npm install --prefix "$RUNNER_TEMP/cypress-testdrive" "cypress@${{ matrix.cypress }}" dd-trace@6.20.1 typescript@5.9.3 --save-exact --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/cypress-testdrive" --depth=0 + - name: Validate original CLI, action API and matching failures with TIA deferred + env: + DDTEST_CYPRESS_NODE_MODULES: ${{ runner.temp }}/cypress-testdrive/node_modules + CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: xvfb-run -a go test -v ./internal/testdrive -run '^(TestPublicCypressTestdrive|TestCypressTypeScriptWrapperTranspilesToCommonJS)$' -failfast -count=1 -timeout=12m + + # Resolve floating dependencies before expanding the matrix. Identical aliases + # share locked fixtures and jobs within this comprehensive run. + cucumber-matrix: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve versions, lock fixtures and select distinct configurations + id: plan + run: | + python3 scripts/ci_cucumber_matrix.py --output "$RUNNER_TEMP/cucumber-fixtures" \ + --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures + if-no-files-found: error + retention-days: 14 + + testdrive-cucumber-workflow: + name: testdrive-cucumber-workflow (${{ matrix.configuration.cucumber }}, ${{ matrix.configuration.shard }}) + needs: cucumber-matrix + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.configuration.node }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures + - name: Install Cucumber testdrive fixture dependencies + run: | + test "$(npm --version)" = '${{ matrix.configuration.npm }}' + echo 'Cucumber ${{ matrix.configuration.version }} covers: ${{ matrix.configuration.aliases }}' + mkdir -p "$RUNNER_TEMP/cucumber-testdrive" + cp "$RUNNER_TEMP/cucumber-fixtures/${{ matrix.configuration.fixture }}"/package*.json "$RUNNER_TEMP/cucumber-testdrive/" + npm ci --prefix "$RUNNER_TEMP/cucumber-testdrive" --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/cucumber-testdrive" --depth=0 + - name: Validate native scenarios, matching failures and source coverage + env: + DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/cucumber-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.configuration.shard }} + run: | + go test -v ./internal/testdrive -run '${{ matrix.configuration.tests }}' -failfast -count=1 -timeout=12m + + testdrive-webdriverio-workflow: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # npm runs every scenario across shards; pnpm checks its layout on one. + # The latest lane also takes the latest tracer: support for a new + # WebdriverIO major lands there first. + include: + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '2/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '3/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '4/4'} + - {package-manager: pnpm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/1'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '1/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '2/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '3/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '4/4'} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Install WebdriverIO testdrive fixture dependencies + run: | + mkdir -p "$RUNNER_TEMP/wdio-testdrive" + printf '{"private":true}\n' > "$RUNNER_TEMP/wdio-testdrive/package.json" + if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + npm exec --yes --package=pnpm@10.20.0 -- pnpm --dir "$RUNNER_TEMP/wdio-testdrive" add --save-dev --save-exact @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} + else + npm install --prefix "$RUNNER_TEMP/wdio-testdrive" @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} --save-exact --no-audit --no-fund + fi + npm ls --prefix "$RUNNER_TEMP/wdio-testdrive" --depth=0 + - name: Validate real browser outcomes with Linux Xvfb supervision + env: + DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/wdio-testdrive/node_modules + DDTEST_CHROME_BINARY: /usr/bin/google-chrome + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: | + command -v xvfb-run + if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$/^passing$' -failfast -count=1 -timeout=8m + else + go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$' -failfast -count=1 -timeout=12m + fi + if [ '${{ matrix.shard }}' = '1/4' ] || [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + go test -v ./internal/framework ./internal/testdrive -run '^TestWebdriverIO' -failfast -count=1 -timeout=5m + fi + + onboarding-evaluator-isolation: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - name: Build fake agent image (no model sessions) + run: | + scripts=skills/ddtest-onboarding-eval/scripts + docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" + echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" + - name: Verify environment, signals and detached process isolation + run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2e8b22a8..423886ff6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,982 +2,186 @@ name: CI on: push: - branches: - - "**" + branches: [main] + pull_request: + workflow_call: + workflow_dispatch: -jobs: - # Shadow mode: source decisions never gate the existing test jobs. Only a - # successful main run may become the baseline for a later source comparison. - integration-plan: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - actions: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Compare integration inputs with passed main evidence - env: - GH_TOKEN: ${{ github.token }} - run: | - python3 scripts/ci_test_plan.py --github-baseline \ - --output "$RUNNER_TEMP/plan.json" --summary "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ci-integration-shadow-plan - path: ${{ runner.temp }}/plan.json - if-no-files-found: error - retention-days: 14 - - test-support: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist +permissions: + contents: read - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 +concurrency: + group: fast-ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true - - name: Test onboarding evaluator - run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' +env: + PYTHONDONTWRITEBYTECODE: '1' + GORACE: atexit_sleep_ms=0 - # install.sh, and with the runner's pwsh install.ps1, against local release files. - - name: Test install scripts - run: python -m unittest discover -s scripts -p 'test_*.py' - - test-shards: +jobs: + plan: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - + outputs: + go: ${{ steps.plan.outputs.go }} + ddto: ${{ steps.plan.outputs.ddto }} + evaluator: ${{ steps.plan.outputs.evaluator }} + integrations: ${{ steps.plan.outputs.integrations }} + playwright: ${{ steps.plan.outputs.playwright }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - # This job already has a compiler-specific cache below. Restoring - # setup-go's cache too collides with read-only module files. - cache: false - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Cache Go modules - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 - - # ddto is its own module, checked by its own workflow; its benchmark - # fixtures are deliberately compact. - - name: Format check - run: | - unformatted="$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" - if [ -n "$unformatted" ]; then - echo "The following files are not formatted correctly:" - echo "$unformatted" - exit 1 - fi - - - name: Vet - run: go vet ./... - - - name: Select Go test shard - run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" - - - name: Configure Datadog Test Optimization - if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} - uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 - with: - languages: go - # Pin upstream Go 1.27 compatibility fixes until they reach a stable - # Orchestrion release. - go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 - api_key: ${{ secrets.DATADOG_API_KEY }} - site: datadoghq.com - service: ddtest - - - name: Run tests - run: go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... - env: - DD_ENV: ci - DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" - - - name: Retain shard coverage - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: unit-coverage-${{ matrix.shard }} - path: coverage.out - if-no-files-found: error - overwrite: true - retention-days: 1 - - # Keep the existing required check name; a skipped, cancelled or failed - # shard/support job must not turn the aggregate green. - test: - if: ${{ always() }} - needs: [test-shards, test-support] - runs-on: ubuntu-latest - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - steps: - - name: Require every test job + fetch-depth: 0 + - name: Select affected checks + id: plan env: - SHARDS_RESULT: ${{ needs.test-shards.result }} - SUPPORT_RESULT: ${{ needs.test-support.result }} + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} run: | - test "$SHARDS_RESULT" = success - test "$SUPPORT_RESULT" = success - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: unit-coverage-* - path: ${{ runner.temp }}/unit-coverage - - name: Merge complete coverage - run: python3 scripts/go_test_shards.py merge --expected 3 --output coverage.out "$RUNNER_TEMP"/unit-coverage/*/coverage.out - - name: Upload coverage report to Datadog - if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} - uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 # v1.0.5 - with: - api_key: ${{ secrets.DATADOG_API_KEY }} - site: datadoghq.com - files: coverage.out - format: go-coverprofile - version: v5.19.0 - - # Dogfooding: runs the `test` suite with ddtest instrument, built from this - # checkout as a release builds it, instead of Orchestrion. Its results reach - # Test Optimization beside those of `test`, under this job's name. - test-ddtest-instrument-shards: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - # This job already has a compiler-specific cache below. Restoring - # setup-go's cache too collides with read-only module files. - cache: false - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - # Its own key, so that its instrumented builds don't replace those of `test`. - - name: Cache Go modules - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 - - - name: Select Go test shard - run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" - - # The release builder, as `make release` runs it for one target: ddtest - # with the runtime sources and tool wrapper that ddtest instrument provides. - - name: Build ddtest instrument from this checkout - run: | - version="dogfood-${GITHUB_SHA::12}" - go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ - -version "v0.0.0-$version" -ldflags "-X github.com/DataDog/ddtest/internal/buildinfo.Version=$version" \ - -target linux/amd64 -o "$RUNNER_TEMP/dist" - mkdir -p "$RUNNER_TEMP/bin" - mv "$RUNNER_TEMP/dist/ddtest-linux-amd64" "$RUNNER_TEMP/bin/ddtest" - echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" - "$RUNNER_TEMP/bin/ddtest" instrument --version - - - name: Run tests - run: ddtest instrument go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... - env: - DD_CIVISIBILITY_ENABLED: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' && 'parent' || 'false' }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_SERVICE: ddtest - DD_ENV: ci - JOB_CHECK_RUN_ID: ${{ job.check_run_id }} - DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" - # `test` uploads this commit's coverage report. - DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED: "false" - - test-ddtest-instrument: - if: ${{ always() }} - needs: test-ddtest-instrument-shards - runs-on: ubuntu-latest - steps: - - name: Require every instrumented shard - env: - SHARDS_RESULT: ${{ needs.test-ddtest-instrument-shards.result }} - run: test "$SHARDS_RESULT" = success - - rspec-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - rspec: "3.12.0" - datadog_ci: "1.31.0" - - rspec: "3.13.2" - datadog_ci: "1.37.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Ruby - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Test RSpec adapter - run: | - gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document - gem install rspec --version "${{ matrix.rspec }}" --no-document - DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ - DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$' - DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$' - - minitest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - minitest: "5.26.0" - datadog_ci: "1.31.0" - - minitest: "6.0.6" - datadog_ci: "1.37.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Ruby - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Test Minitest adapter - run: | - gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document - gem install minitest --version "${{ matrix.minitest }}" --no-document - gem install rake --version "13.2.1" --no-document - DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ - DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$' - - pytest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - pytest: "8.4.2" - ddtrace: "4.11.0" - - pytest: "9.1.1" - ddtrace: "4.14.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Test pytest adapter - run: | - python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" pytest-cov pytest-xdist - DDTEST_PYTHON_BINARY="$(command -v python)" \ - go test -v ./internal/compatibility -run '^TestPyTest(AdapterIntegration|SuiteModeSelectorBatches)$' - DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ - go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$' - DDTEST_PYTEST_PYTHON="$(command -v python)" \ - go test -v ./internal/testdrive -run '^TestPytest(RealValidation|CoverageExecutesOnlyInsideCopy|LoggingExecutesOnlyInsideCopy|RejectsEditableImportsFromOriginalCheckout|CoverageDataExecutesOnlyInsideCopy|IgnoredConfigurationUsesRoot|ObserverKeepsConcurrentDuplicateExecutions|PreflightReportsSelectedEnvironment)$' - - javascript-platform-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - dd_trace: ["5.111.0", "6.15.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + python3 scripts/ci_test_plan.py plan --base "$BASE" \ + --output "$RUNNER_TEMP/plan.json" --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - node-version: "22" - - - name: Test JavaScript platform - run: | - dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" - npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" - DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestJavaScript(Platform|ActionPreload)Integration$' + name: pr-test-plan + path: ${{ runner.temp }}/plan.json - jest-compatibility: + unit: + needs: plan + if: needs.plan.outputs.go == 'true' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - jest: ["27.5.1", "28.1.3", "29.4.3", "29.7.0", "30.5.1"] - + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Jest adapter - run: | - jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" - npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" dd-trace@5.86.0 - DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestJest(Adapter|Testdrive)Integration$' - DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestJestCheckOnlyIntegration$' - - vitest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.5"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + fetch-depth: 0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - node-version: "22" - - - name: Test Vitest adapter - run: | - vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" - # npm 10.9.8 crashes while resolving Vitest's peer dependencies. - # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. - npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "@vitest/coverage-v8@${{ matrix.vitest }}" "dd-trace@5.132.0" - DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' - DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestVitest(Configuration|ValidationArguments|MultiProjectRejectionKeepsConfigurationEvidence|RepeatedRunOptionIsNormalized)Integration$' + node-version: '24' + - name: Test affected Go packages and their dependents + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + run: python3 scripts/ci_test_plan.py run --base "$BASE" --race - mocha-compatibility: + scripts: + needs: plan runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"] - + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - node-version: "22" - - - name: Test Mocha adapter - run: | - mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" - npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" - DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestMocha.*Integration$' - DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestMochaNativeObserverIntegration$' - - cypress-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + python-version: '3.12' + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + if: needs.plan.outputs.evaluator == 'true' with: go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Cypress adapter - run: | - cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}" - npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}" - DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ - DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ - DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ - go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$' - - playwright-compatibility: + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: needs.plan.outputs.evaluator == 'true' + with: + node-version: '22' + - name: Install evaluator package-manager fixtures + if: needs.plan.outputs.evaluator == 'true' + run: npm install --global yarn@1.22.22 pnpm@12.14.0 + - name: Test CI and install scripts + run: python3 -m unittest discover -s scripts -p 'test_*.py' + - name: Test changed onboarding evaluator + if: needs.plan.outputs.evaluator == 'true' + run: python3 -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' + + instrument: + needs: plan + if: needs.plan.outputs.go == 'true' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"] - + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Playwright adapter + - name: Build release instrumentation and run its smoke packages env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + DD_CIVISIBILITY_ENABLED: 'false' run: | - playwright_dir="${RUNNER_TEMP}/playwright-${{ matrix.playwright }}" - npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}" - DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \ - DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$' + go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ + -version v0.0.0-pr -target linux/amd64 -o "$RUNNER_TEMP/dist" + "$RUNNER_TEMP/dist/ddtest-linux-amd64" instrument go test -race ./internal/buildinfo ./internal/errcode - cucumber-compatibility: + instrumentation-unit: + needs: plan + if: needs.plan.outputs.ddto == 'true' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"] - + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Cucumber adapter + cache-dependency-path: ddto/go.mod + - name: Audit and test instrumentation units on the current toolchain + working-directory: ddto run: | - cucumber_dir="${RUNNER_TEMP}/cucumber-${{ matrix.cucumber }}" - npm install --prefix "${cucumber_dir}" "@cucumber/cucumber@${{ matrix.cucumber }}" - DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ - DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ - DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ - go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$' - DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestCucumberSupportMetadataIntegration$' - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: golangci-lint - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 - with: - version-file: .golangci-lint-version - args: --timeout=5m - - jest-testdrive-workflow: - name: jest-testdrive-workflow (${{ matrix.node }}, ${{ matrix.jest }}, ${{ matrix.group }} ${{ matrix.shard }}) + python3 scripts/upstream.py verify + python3 scripts/dependency_boundary.py + python3 -m unittest discover -s scripts -p 'test_*.py' + go vet ./... + packages=$(go list ./... | grep -v '/integration$') + go test -short -race $packages + + integration: + needs: plan + if: needs.plan.outputs.integrations != '[]' runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: - # Per runtime: TestPublicJestTestdrive over two shards, plus two groups of - # configuration tests of similar length; together they are the full set. - include: - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} - # Unpinned: catches framework releases our pinned versions predate. - - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} + configuration: ${{ fromJSON(needs.plan.outputs.integrations) }} + name: smoke (${{ matrix.configuration.id }}) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: matrix.configuration.runtime == 'node' with: - node-version: ${{ matrix.node }} - - name: Install real Jest and tracer - run: | - npm install --prefix "${RUNNER_TEMP}/jest-testdrive" jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 - npm ls --prefix "${RUNNER_TEMP}/jest-testdrive" --depth=0 - - name: Validate paired outcomes, all feature probes and cleanup - env: - DDTEST_JEST_NODE_MODULES: ${{ runner.temp }}/jest-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(${{ matrix.tests }})$' -count=1 -timeout=12m - - windows-probe-diagnostics: - runs-on: windows-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - name: Test Windows probe filesystem diagnostics - run: go test ./internal/testdrive -run '^(TestProbePublicationExplainsWindowsFilesystemErrors|TestReportsCanReplaceExistingDestinationTwice)$' -count=1 - - - testdrive-language-workflow: - runs-on: ubuntu-latest - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - framework: [pytest] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod + node-version: '24' - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + if: matrix.configuration.runtime == 'python' with: python-version: '3.12' - - name: Run Python testdrive - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - run: go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives/${{ matrix.framework }}$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: testdrive-${{ matrix.framework }} - path: ${{ runner.temp }}/ddtest-workflow-reports - - ruby-testdrive: - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - include: - - name: pinned-minitest-6 - datadog_ci: "1.40.0" - rspec: "3.13.2" - minitest: "6.0.6" - rake: "13.4.2" - - name: pinned-minitest-5 - datadog_ci: "1.40.0" - rspec: "3.13.2" - minitest: "5.27.0" - rake: "13.4.2" - - name: latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + if: matrix.configuration.runtime == 'ruby' with: ruby-version: '3.4' - - name: Install Ruby validation dependencies - env: - DATADOG_CI: ${{ matrix.datadog_ci }} - RSPEC: ${{ matrix.rspec }} - MINITEST: ${{ matrix.minitest }} - RAKE: ${{ matrix.rake }} - run: | - install() { if [ -n "$2" ]; then gem install "$1" --version "$2" --no-document; else gem install "$1" --no-document; fi; } - install datadog-ci "$DATADOG_CI" - install rspec "$RSPEC" - install minitest "$MINITEST" - install rake "$RAKE" - gem install activesupport --conservative --no-document - gem list | grep -E '^(datadog-ci|rspec-core|minitest|rake|activesupport) ' - - name: Run Ruby testdrive integration tests - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_RUN_RUBY_INTEGRATION_TEST: '1' - DDTEST_RUN_RUBY_FORK_INTEGRATION_TEST: '1' - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - run: | - go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives$/^(rspec|minitest)$' -count=1 -timeout=20m - go test -v ./internal/testdrive -run '^TestRuby.+Integration$' -skip '^TestRubyValidationIntegration$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ruby-testdrive-${{ matrix.name }} - path: ${{ runner.temp }}/ddtest-workflow-reports - if-no-files-found: ignore - retention-days: 7 - - java-testdrive: - name: java-testdrive (${{ matrix.name }}, ${{ matrix.java }}, ${{ matrix.tracer }}, ${{ matrix.shard }}) - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - # Each scenario runs a full paired validation with real Maven or Gradle builds; - # the scenarios are spread over shards (DDTEST_TEST_SHARD). - name: [pinned, latest] - shard: ['1/4', '2/4', '3/4', '4/4'] - include: - - name: pinned - java: '17' - tracer: '1.67.1' - - name: latest - java: '21' - tracer: latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + if: matrix.configuration.runtime == 'java' || matrix.configuration.runtime == 'gradle' with: distribution: temurin - java-version: ${{ matrix.java }} - # The Maven fixtures are generated by the test, so the repository has no - # pom.xml for setup-java's cache key; key the cache on the test instead. - - name: Cache Maven dependencies - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ matrix.java }}-${{ hashFiles('internal/testdrive/java_validation_integration_test.go') }} - restore-keys: | - ${{ runner.os }}-maven-${{ matrix.java }}- - # The Gradle fixtures run the gradle on PATH, as a CI job with this - # action does; it also caches the Gradle user home. + java-version: '17' - uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 + if: matrix.configuration.runtime == 'gradle' with: gradle-version: '8.10.2' - - name: Run Java testdrive integration tests - env: - DDTEST_RUN_JAVA_INTEGRATION_TEST: '1' - DDTEST_JAVA_TRACER_VERSION: ${{ matrix.tracer }} - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - DDTEST_TEST_SHARD: ${{ matrix.shard }} - # Every validation starts Maven many times; short-lived Maven JVMs - # start faster without the optimizing compiler. Both runs inherit it. - MAVEN_OPTS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 - run: go test -v ./internal/testdrive -run '^TestJavaValidationIntegration$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + if: matrix.configuration.runtime == 'node' with: - name: java-testdrive-${{ matrix.name }}-${{ strategy.job-index }} - path: ${{ runner.temp }}/ddtest-workflow-reports - if-no-files-found: ignore - retention-days: 7 - - testdrive-vitest-workflow: - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - include: - - vitest: '3.2.7' - tracer: '5.132.0' - - vitest: '4.1.11' - tracer: '6.21.0' - - vitest: '5.0.5' - tracer: '6.21.0' - - vitest: 'latest' - tracer: '6.21.0' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Resolve Vitest version - run: | - version=$(npm view "vitest@${{ matrix.vitest }}" version) - echo "Vitest $version" - echo "DDTEST_VITEST_VERSION=$version" >> "$GITHUB_ENV" - - name: Validate Vitest CLI and Node API paths against local intake - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_VITEST_TRACER_VERSION: ${{ matrix.tracer }} - run: go test -v ./internal/testdrive -run '^TestPublicVitestRunnerTestdrive$' -count=1 -timeout=15m - - onboard-apply-workflow: + path: | + ~/.npm + ~/.cache/Cypress + key: smoke-${{ runner.os }}-${{ matrix.configuration.id }}-${{ hashFiles('scripts/ci_test_plan.json') }} + restore-keys: smoke-${{ runner.os }}-${{ matrix.configuration.id }}- + - name: Run selected smoke and require every named scenario + run: python3 scripts/ci_smoke.py --id '${{ matrix.configuration.id }}' + + playwright: + needs: plan + if: needs.plan.outputs.playwright == 'true' runs-on: ubuntu-latest timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Validate generated workflow edits with actionlint - run: | - go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 - DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/onboard -run '^TestGeneratedSetupWorkflow.*Actionlint$' -count=1 - - name: Install apply fixture dependencies - run: npm install --prefix "$RUNNER_TEMP/apply-testdrive" vitest@4.1.11 mocha@11.7.6 nyc@17.1.0 dd-trace@6.20.1 @cucumber/cucumber@12.2.0 @wdio/cli@9.20.0 @wdio/local-runner@9.20.0 @wdio/mocha-framework@9.20.0 --no-audit --no-fund - - name: Apply setup and validate against local intake - env: - DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_VITEST_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_CHROME_BINARY: /usr/bin/google-chrome - run: DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/testdrive -run '^TestPublic(OnboardApply|MochaApply|WebdriverIOApply|CucumberApply)$' -count=1 -timeout=8m - - testdrive-mocha-workflow: - runs-on: ubuntu-latest - timeout-minutes: 12 - strategy: - fail-fast: false - matrix: - mocha: ['11.7.6', 'latest'] - shard: ['1/2', '2/2'] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Install Mocha testdrive fixture dependencies - run: | - npm install --prefix "$RUNNER_TEMP/mocha-testdrive" "mocha@${{ matrix.mocha }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.21.0 --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/mocha-testdrive" --depth=0 - - name: Validate CLI, programmatic runner, existing failures and coverage gate - env: - DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/mocha-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(TestPublicMochaTestdrive|TestMochaProbeCoverageThresholdControls|TestMochaDelayedCommandUsesSharedTimeout|TestMochaPreflightResolvesSelectedPackage)$' -count=1 -timeout=10m - - playwright-latest: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.resolve.outputs.version }} - steps: - - id: resolve - run: echo "version=$(npm view @playwright/test version)" >> "$GITHUB_OUTPUT" - - testdrive-playwright-workflow: - needs: playwright-latest - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - playwright: ['1.55.1', 'latest'] - shard: ['1/2', '2/2'] - # The official image ships the browsers and their system packages for this - # exact Playwright release, so nothing is downloaded or installed per run. container: - image: mcr.microsoft.com/playwright:v${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }}-noble - env: - PLAYWRIGHT_VERSION: ${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - # The workspace is owned by the runner user, not the container's root; - # go build reads it with git for VCS stamping. - - name: Trust the workspace for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Isolate Playwright fixture caches - run: echo "npm_config_cache=$RUNNER_TEMP/playwright-testdrive/cache" >> "$GITHUB_ENV" - - name: Install Playwright testdrive fixture dependencies - run: | - npm install --prefix "$RUNNER_TEMP/playwright-testdrive" "@playwright/test@$PLAYWRIGHT_VERSION" dd-trace@6.20.1 --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/playwright-testdrive" --depth=0 - - name: Validate browser execution, existing failures and supervised cleanup - env: - DDTEST_PLAYWRIGHT_NODE_MODULES: ${{ runner.temp }}/playwright-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(TestPublicPlaywrightTestdrive|TestPlaywrightDiscoveryOwnsImportSideEffects|TestPlaywrightDiscoveryKeepsConfiguredReportsOutOfTheCheckout)$' -count=1 -timeout=12m - - testdrive-cypress-workflow: - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # Scenarios are split by integration_shard_test.go; each runs exactly - # once per Cypress version. - cypress: ['15.1.0', 'latest'] - shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] + image: mcr.microsoft.com/playwright:v1.55.1-noble steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 @@ -986,178 +190,74 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' - - name: Resolve Cypress version - id: cypress - run: echo "version=$(npm view "cypress@${{ matrix.cypress }}" version | tail -n 1)" >> "$GITHUB_OUTPUT" - # The Cypress binary downloads during npm install; reuse it per release. - - name: Cache Cypress binary - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: ${{ runner.temp }}/cypress-testdrive-cache - key: ${{ runner.os }}-cypress-binary-${{ steps.cypress.outputs.version }} - - name: Install Cypress testdrive fixture dependencies - env: - CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache - run: | - npm install --prefix "$RUNNER_TEMP/cypress-testdrive" "cypress@${{ matrix.cypress }}" dd-trace@6.20.1 typescript@5.9.3 --save-exact --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/cypress-testdrive" --depth=0 - - name: Validate original CLI, action API and matching failures with TIA deferred - env: - DDTEST_CYPRESS_NODE_MODULES: ${{ runner.temp }}/cypress-testdrive/node_modules - CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: xvfb-run -a go test -v ./internal/testdrive -run '^(TestPublicCypressTestdrive|TestCypressTypeScriptWrapperTranspilesToCommonJS)$' -failfast -count=1 -timeout=12m + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: python3 scripts/ci_smoke.py --id playwright - testdrive-cucumber-workflow: + evaluator-isolation: + needs: plan + if: needs.plan.outputs.evaluator == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # 10.9.0 and 13.3.0 run every scenario across shards; 8.11.1 the floor check. - include: - - {cucumber: '8.11.1', shard: '1/1'} - - {cucumber: '10.9.0', shard: '1/4'} - - {cucumber: '10.9.0', shard: '2/4'} - - {cucumber: '10.9.0', shard: '3/4'} - - {cucumber: '10.9.0', shard: '4/4'} - - {cucumber: '13.3.0', shard: '1/4'} - - {cucumber: '13.3.0', shard: '2/4'} - - {cucumber: '13.3.0', shard: '3/4'} - - {cucumber: '13.3.0', shard: '4/4'} - - {cucumber: 'latest', shard: '1/4'} - - {cucumber: 'latest', shard: '2/4'} - - {cucumber: 'latest', shard: '3/4'} - - {cucumber: 'latest', shard: '4/4'} + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - node-version: '24' - - name: Install Cucumber testdrive fixture dependencies - run: | - npm install --prefix "$RUNNER_TEMP/cucumber-testdrive" "@cucumber/cucumber@${{ matrix.cucumber }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.20.1 @istanbuljs/esm-loader-hook@0.3.0 tsx@4.23.15 --save-exact --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/cucumber-testdrive" --depth=0 - - name: Validate native scenarios, matching failures and source coverage - env: - DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/cucumber-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} + python-version: '3.12' + - name: Build offline evaluator image run: | - test_filter='^TestPublicCucumberTestdrive$' - if [ '${{ matrix.cucumber }}' = '8.11.1' ]; then test_filter='^TestPublicCucumberTestdrive/passing$'; fi - go test -v ./internal/testdrive -run "$test_filter" -failfast -count=1 -timeout=12m + scripts=skills/ddtest-onboarding-eval/scripts + docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" + echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" + - run: python3 -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v - testdrive-webdriverio-workflow: + lint: + needs: plan + if: needs.plan.outputs.go == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # npm runs every scenario across shards; pnpm checks its layout on one. - # The latest lane also takes the latest tracer: support for a new - # WebdriverIO major lands there first. - include: - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '2/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '3/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '4/4'} - - {package-manager: pnpm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/1'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '1/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '2/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '3/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '4/4'} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Install WebdriverIO testdrive fixture dependencies - run: | - mkdir -p "$RUNNER_TEMP/wdio-testdrive" - printf '{"private":true}\n' > "$RUNNER_TEMP/wdio-testdrive/package.json" - if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - npm exec --yes --package=pnpm@10.20.0 -- pnpm --dir "$RUNNER_TEMP/wdio-testdrive" add --save-dev --save-exact @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} - else - npm install --prefix "$RUNNER_TEMP/wdio-testdrive" @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} --save-exact --no-audit --no-fund - fi - npm ls --prefix "$RUNNER_TEMP/wdio-testdrive" --depth=0 - - name: Validate real browser outcomes with Linux Xvfb supervision - env: - DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/wdio-testdrive/node_modules - DDTEST_CHROME_BINARY: /usr/bin/google-chrome - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: | - command -v xvfb-run - if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$/^passing$' -failfast -count=1 -timeout=8m - else - go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$' -failfast -count=1 -timeout=12m - fi - if [ '${{ matrix.shard }}' = '1/4' ] || [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - go test -v ./internal/framework ./internal/testdrive -run '^TestWebdriverIO' -failfast -count=1 -timeout=5m - fi - - onboarding-evaluator-isolation: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - name: Format check + run: test -z "$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 with: - python-version: "3.12" - - name: Build fake agent image (no model sessions) - run: | - scripts=skills/ddtest-onboarding-eval/scripts - docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" - echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" - - name: Verify environment, signals and detached process isolation - run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v + version-file: .golangci-lint-version + args: --timeout=5m - # Publish exactly the source plan whose full integration matrix just passed. - # A skipped, failed or cancelled matrix must never certify those inputs. - integration-evidence: + # Preserve the required check name and reject every failed or unexpectedly + # skipped selected job. Only the planner may intentionally omit a check. + test: if: ${{ always() }} - needs: [integration-plan, rspec-compatibility, minitest-compatibility, - pytest-compatibility, javascript-platform-compatibility, jest-compatibility, - vitest-compatibility, mocha-compatibility, cypress-compatibility, - playwright-compatibility, cucumber-compatibility, jest-testdrive-workflow, - testdrive-language-workflow, ruby-testdrive, java-testdrive, - testdrive-vitest-workflow, onboard-apply-workflow, testdrive-mocha-workflow, - playwright-latest, testdrive-playwright-workflow, testdrive-cypress-workflow, - testdrive-cucumber-workflow, testdrive-webdriverio-workflow] + needs: [plan, unit, scripts, instrument, instrumentation-unit, integration, playwright, evaluator-isolation, lint] runs-on: ubuntu-latest - timeout-minutes: 5 steps: - - name: Require the complete integration matrix + - name: Require every selected check env: RESULTS: ${{ toJSON(needs) }} run: | python3 - <<'PY' import json, os - failed = {name: job["result"] for name, job in json.loads(os.environ["RESULTS"]).items() - if job["result"] != "success"} - if failed: - raise SystemExit(f"Integration evidence incomplete: {failed}") + results = json.loads(os.environ['RESULTS']) + if results['plan']['result'] != 'success': + raise SystemExit('Test selection failed') + plan = results['plan']['outputs'] + required = {'plan', 'scripts'} + if plan['go'] == 'true': + required.update(['unit', 'instrument', 'lint']) + if plan['ddto'] == 'true': + required.add('instrumentation-unit') + if plan['evaluator'] == 'true': + required.add('evaluator-isolation') + if json.loads(plan['integrations']): + required.add('integration') + if plan['playwright'] == 'true': + required.add('playwright') + bad = {name: job['result'] for name, job in results.items() + if job['result'] != ('success' if name in required else 'skipped')} + if bad: + raise SystemExit(f'Selected checks incomplete: {bad}') PY - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - if: github.ref == 'refs/heads/main' - with: - name: ci-integration-shadow-plan - path: ${{ runner.temp }}/integration-evidence - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - if: github.ref == 'refs/heads/main' - with: - name: ci-integration-source-evidence-v1 - path: ${{ runner.temp }}/integration-evidence/plan.json - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/go-instrumentation.yml b/.github/workflows/go-instrumentation.yml index 1df8ba4ea..2b070016f 100644 --- a/.github/workflows/go-instrumentation.yml +++ b/.github/workflows/go-instrumentation.yml @@ -1,19 +1,14 @@ name: Go instrumentation # The Go instrumentation that ddtest instrument runs is its own module in ddto/. -# Its suite runs from there, with ddtest's executable as the driver. Every pull -# request and push to main runs this workflow, so that its result can be a -# required check: when a run already verified the files that the suite depends -# on (see the plan job's inputs), the groups do not run again. +# Its complete matrix runs for releases, nightly upstream checks and manual +# dispatch. Pull requests use the native unit checks in ci.yml instead. # -# Pull requests and pushes test the dd-trace-go and Orchestrion versions in -# ddto/scripts/supported_versions.json. The nightly run tests the versions that -# the module proxy has then, and fails when that file is out of date. The -# release workflow calls this one with compatibility: full first: those +# Manual runs use the dd-trace-go and Orchestrion versions in +# ddto/scripts/supported_versions.json by default. The nightly run tests the +# versions that the module proxy has then, and fails when that file is out of +# date. The release workflow calls this one with compatibility: full: those # versions, each one run even when another has the same edited SDK packages. on: - push: - branches: [main] - pull_request: schedule: - cron: '17 3 * * *' workflow_call: @@ -31,8 +26,7 @@ on: default: recorded permissions: contents: read -# A new push to a pull request replaces its running matrix. Other events never -# share a group, so main and scheduled runs always finish. +# Independent release, nightly and manual runs never share a group. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 770e77fb6..23e49da0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,14 +43,11 @@ jobs: GH_TOKEN: ${{ steps.octo-sts.outputs.token }} run: bash .github/scripts/release.sh validate - # Before a release, the Go instrumentation suite runs every dd-trace-go and - # Orchestrion version that ddtest instrument supports, as the module proxy - # has them now, so that no release ships without them passing. + # Every release waits for the complete framework, platform and Go instrumentation + # matrices, including every currently supported SDK and Orchestrion version. compatibility: needs: authorize - uses: ./.github/workflows/go-instrumentation.yml - with: - compatibility: full + uses: ./.github/workflows/ci-full.yml permissions: contents: read actions: read diff --git a/Makefile b/Makefile index f239804de..90979fb73 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := build -.PHONY: clean fmt vet lint test build install run release +.PHONY: clean fmt vet lint test test-fast build install run release VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) GOBIN ?= $(shell go env GOBIN) @@ -16,6 +16,8 @@ lint: vet golangci-lint run --timeout=5m test: go test ./... +test-fast: + python3 scripts/ci_test_plan.py run --base "$(or $(TEST_BASE),origin/main)" build: test lint go build -ldflags="$(LDFLAGS)" -o ddtest main.go install: diff --git a/ddto/AGENTS.md b/ddto/AGENTS.md index 45484a21c..7b697da1f 100644 --- a/ddto/AGENTS.md +++ b/ddto/AGENTS.md @@ -360,7 +360,7 @@ maintenance guides. Before proposing a change: - `docs/benchmarks.md` and `docs/results/**` are generated by the benchmark scripts; never edit them by hand. See [the benchmark guide](docs/build-benchmarks.md). -- CI reuses a passed run when a change leaves the suite's inputs unchanged, - such as one limited to `docs/` and `README.md`; other changes run the full - compatibility matrix +- Pull requests run native units on the current Go toolchain and a release + binary smoke. The full compatibility matrix gates releases and runs nightly; + it is also available by manual dispatch ([details](docs/validation.md#go-instrumentation-workflow)). diff --git a/ddto/docs/validation.md b/ddto/docs/validation.md index c3b0d32d5..2d50f7f65 100644 --- a/ddto/docs/validation.md +++ b/ddto/docs/validation.md @@ -207,7 +207,7 @@ families and tip: | Linux | 1.25 | Native Mini suite in normal and `-race` modes; `GOTOOLCHAIN=local` prevents an upgrade | | Linux | 1.26 | Complete SDK differential suite | | Linux | 1.27 | Complete SDK differential suite in normal and `-race` modes | -| Linux | tip | Native Mini suite and manual SDK span-copy cases; source commit recorded before building Go. Pull requests, nightly and manual dispatch | +| Linux | tip | Native Mini suite and manual SDK span-copy cases; source commit recorded before building Go | | macOS | 1.27 | Normal suite | | Windows | 1.27 | Normal suite | @@ -235,21 +235,22 @@ report step requires the independent Orchestrion reference and exports feature outcomes, counts and timings. Logs and JSON reports are uploaded even when a test fails. -Every pull request and push to `main`, a nightly schedule and manual dispatch -start the workflow, so that its last job, Complete compatibility evidence, can -be a required check. A new push to a pull request cancels its previous run. The -suite depends on this directory, except `docs/` and `README.md`, and on what -builds and runs ddtest's executable: ddtest's `main.go`, `go.mod`, `go.sum`, -`internal/cmd/instrument.go`, install scripts and the workflow. A pull request -or a push to `main` whose copies of those files already passed a complete run of -a branch in this repository reuses that run instead of repeating the matrix: -typically a pull request that changes none of them, or the squash merge of an -up-to-date pull request. It uploads that run's coverage of ddto's packages to -Datadog for its own commit, since Datadog counts lines without a report as -uncovered. Nightly and manual runs always run the matrix. Pull requests, nightly -and manual runs also test Go tip, which changes independently of this -repository: a pull request can fail on tip without causing it. A push to `main` -that runs the matrix itself skips tip. +The full matrix runs nightly, on manual dispatch, and as part of the +[Full compatibility](../../.github/workflows/ci-full.yml) gate before a release. +Release builds wait for its complete compatibility evidence on the same commit. +Manual Go instrumentation runs default to recorded SDK/Orchestrion versions; +`compatibility: full` and release runs resolve every currently supported version. + +Pull requests and pushes to `main` use the smaller native checks in +[CI](../../.github/workflows/ci.yml). When instrumentation inputs change, these +audit the sources and dependency boundary, vet the module and run its local +packages with `-short -race` on the current toolchain. Code changes also build +the release executable and instrument two small packages. They omit the +integration package, SDK differential comparisons, other OS/toolchain versions, +Go tip and aggregate coverage. Documentation changes omit runtime checks. +[PR selection](../../docs/ci-test-selection.md) describes the active policy and +the contributor command. A green PR validates this smaller contract; release +and nightly runs provide comprehensive compatibility evidence. ### Parallel test groups diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index 4bb10de85..634491800 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -1,62 +1,99 @@ -# Integration test selection - -The CI workflow runs an integration input planner in **shadow mode**. Every -existing test job still runs. The job summary reports which suites have changed -source inputs and which are candidates for reusing earlier evidence. The JSON -artifact, `ci-integration-shadow-plan`, retains every source digest, each suite's -fingerprint and changed paths, and the baseline's run and revision. - -The baseline is an artifact from one of the ten most recent successful `CI` -push runs on `main` in this repository. The workflow publishes that artifact -only after every integration matrix passes. Failed, skipped or cancelled jobs -cannot publish evidence. Missing, expired, incompatible or unavailable baseline -artifacts make every suite run in the plan. - -## Input ownership - -[`scripts/ci_test_plan.json`](../scripts/ci_test_plan.json) declares each suite's -jobs, owned input paths and dependencies on other suites. Shared and -unclassified files affect every suite, including changes to the module files, -CLI entry point, workflow, planner and manifest. Only explicitly listed -documentation paths are ignored; embedded instructions are inputs. -`additional_inputs` adds a consumer without narrowing ownership: generic -onboarding helpers still affect every suite. - -For example, a change to `internal/testdrive/python_validation.go` affects the -Python suite. A JavaScript platform change affects every JavaScript suite and -onboarding. Some Jest-named files implement shared validation and feature -probing; those files conservatively affect every suite. Fixtures without an -explicit owner also affect every suite. - -Content, path and executable/symlink type are fingerprinted. Modification times -do not matter. Added, deleted and renamed inputs invalidate their consumers. -The planner snapshots tracked and non-ignored untracked files, so local previews -include edits without requiring a commit. Keep preview outputs outside the -checkout. - -## Inspect a local plan - -The planner needs Python 3 and Git, with no third-party Python packages: +# Faster PR and contributor checks + +The `CI` workflow actively selects checks for pull requests and pushes to `main`. +It runs native Go tests once, without global coverage or SDK weaving. Affected +frameworks get a pinned Linux smoke, rather than the complete historical-version +and scenario matrices. This is the PR acceptance policy, with no shadow phase. + +`Full compatibility` retains the complete framework matrices, both instrumented +root Go test paths, coverage, and the Go instrumentation OS/toolchain/SDK matrix. +Every release waits for it on the commit being released. Maintainers can also +run it from Actions → Full compatibility → Run workflow on any branch. The +existing nightly Go instrumentation matrix continues to catch upstream changes. +The full framework matrix has no recurring schedule; it runs for releases and +manual dispatches. + +## Local feedback + +From the repository root, with Python 3 and the Go version in `go.mod`: ```sh -python3 scripts/ci_test_plan.py --output /tmp/plan.json --summary /tmp/plan.md -# Compare a second snapshot with the first one: -python3 scripts/ci_test_plan.py --baseline /tmp/plan.json --output /tmp/next-plan.json +make test-fast # compare with the merge base of origin/main +make test-fast TEST_BASE= # choose another base ``` -Use `--github-baseline --repository DataDog/ddtest` instead of `--baseline` to -read passed main evidence with an authenticated GitHub CLI. The artifact records -the original source snapshot, before test setup modifies module files. +This includes staged, unstaged, deleted, renamed and non-ignored untracked files. +A production Go edit selects its package and every transitive importer, including +imports made only by tests. A test-only edit selects its own package. Non-Go +inputs (including embedded assets), removed packages and unknown history select +all root packages. Go's own build and test caches remain enabled. `make test` +still runs the whole root module. The instrumentation in `ddto/` is a separate +module; its local checks are described in [its guide](../ddto/AGENTS.md#changing-this-repository). + +Inspect the same integration selection that CI uses: -## Before enabling reuse +```sh +python3 scripts/ci_test_plan.py plan --base origin/main --output /tmp/pr-plan.json +cat /tmp/pr-plan.json +``` + +To execute a selected framework smoke, install its runtime and dependencies +listed in `scripts/ci_test_plan.json`, then run, for example: + +```sh +python3 scripts/ci_smoke.py --id cucumber +``` -`source_unchanged` is not permission to skip a test. Dependencies installed by -the fixtures, floating versions such as `latest`, Go and language toolchains, -runner/browser identities and execution flags must also be resolved and checked. -The plan explicitly records `external_inputs_verified: false` until that work is -implemented. The current artifact certifies source inputs only. +The smoke command installs its npm fixture in a temporary directory and removes +it afterwards. Ruby smokes install the listed gems in the active Ruby environment. +Browser checks need the CI browser/Xvfb environment; Playwright uses the pinned +official image in `ci.yml`. These checks use controlled local fixtures without +Datadog credentials. + +## Selection and acceptance + +[`scripts/ci_test_plan.json`](../scripts/ci_test_plan.json) owns framework paths, +embedded instructions/scripts, JavaScript dependencies and smoke recipes. +Python-only changes select Python; JavaScript platform changes also select all +JavaScript frameworks. Shared or unclassified production inputs select every +PR smoke. Instrumentation module changes select its units and the root/release +smoke without unrelated framework integrations. Unit-test-only edits omit +external smokes; integration-test edits still select their framework. +Documentation-only changes omit runtime jobs; evaluator-only changes run the +evaluator checks. The small Python suite tests CI and installer scripts on every +PR. Missing base history selects all PR checks, never an empty plan. + +Most framework smokes exercise a passing customer fixture and an existing +failure. Python, Ruby, Java and Vitest use their existing validation journeys; +Java covers both Maven and Gradle. PRs use one pinned framework/tracer/runtime +combination per journey. Older versions, `latest`, remaining scenarios, SDK parity +and platform differences are checked by Full compatibility. + +The `pr-test-plan` artifact and job summary show the selection. Every smoke +requires explicit named Go test cases to report `pass`: a filtered, missing or +skipped case fails. The required `test` check rejects failed, cancelled or +unexpectedly skipped selected jobs. The existing required `lint` check remains. +New production paths default to all smokes; add narrower ownership only when its +framework boundary is known and tested. + +## Deduplicating resolved Cucumber configurations + +The full suite resolves `latest` once and creates a dependency lockfile per exact +Cucumber version before expanding its jobs. Requested labels share a job only +when the resolved version, complete dependency graph, exact Node/npm versions, +test filter and shard all match. For example, when `latest` resolves to the +pinned `13.3.0`, its four duplicate jobs collapse, reducing 13 requested lanes +to 9. A new latest release creates separate jobs again. The `8.11.1` floor smoke +remains distinct from full scenario lanes. + +Workers download the planned fixture, verify npm's version and use `npm ci`. +Registry or lockfile failures fail planning. To inspect a live resolution with +Node 24, npm and registry access, use a fresh directory: + +```sh +python3 scripts/ci_cucumber_matrix.py --output /tmp/cucumber-ci-fixtures \ + --summary /tmp/cucumber-ci-matrix.md +``` -Review shadow decisions against complete runs, especially edits to shared -helpers, embedded scripts and new fixtures. Keep the full integration matrix -while validating the ownership rules. Any later reuse must preserve required -checks and attach the original passed run's provenance to the current result. +Inspect `matrix.json`, the summary and each version's `package-lock.json`. The +summary lists all requested aliases represented by each remaining job. diff --git a/internal/releaseworkflow/ci_shards_test.go b/internal/releaseworkflow/ci_shards_test.go index 15d7d94e8..4ff48d3c6 100644 --- a/internal/releaseworkflow/ci_shards_test.go +++ b/internal/releaseworkflow/ci_shards_test.go @@ -26,7 +26,7 @@ type ciTestJob struct { func ciTestJobs(t *testing.T) map[string]ciTestJob { t.Helper() - data, err := os.ReadFile("../../.github/workflows/ci.yml") + data, err := os.ReadFile("../../.github/workflows/ci-full.yml") require.NoError(t, err) var workflow struct{ Jobs map[string]ciTestJob } require.NoError(t, yaml.Unmarshal(data, &workflow)) diff --git a/internal/testdrive/preflight_review_test.go b/internal/testdrive/preflight_review_test.go index 47f681e99..6557b1ae3 100644 --- a/internal/testdrive/preflight_review_test.go +++ b/internal/testdrive/preflight_review_test.go @@ -152,7 +152,7 @@ func TestRecommendationsUseRegistryTagsAndFlagUnknownMajors(t *testing.T) { } func TestJestSupportRulesCoverCICompatibilityMatrix(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/ci.yml") + data, err := os.ReadFile("../../.github/workflows/ci-full.yml") require.NoError(t, err) var workflow struct { Jobs map[string]struct { diff --git a/internal/testdrive/vitest_review_test.go b/internal/testdrive/vitest_review_test.go index 24033b506..7b17c1754 100644 --- a/internal/testdrive/vitest_review_test.go +++ b/internal/testdrive/vitest_review_test.go @@ -208,7 +208,7 @@ func TestVitestRejectedCommandPublishesCurrentFailure(t *testing.T) { } func TestVitestCIMatrixUsesCompatibleTracerFloors(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/ci.yml") + data, err := os.ReadFile("../../.github/workflows/ci-full.yml") require.NoError(t, err) var ci struct { Jobs map[string]struct { diff --git a/scripts/ci_cucumber_matrix.py b/scripts/ci_cucumber_matrix.py new file mode 100644 index 000000000..0ebefebd1 --- /dev/null +++ b/scripts/ci_cucumber_matrix.py @@ -0,0 +1,116 @@ +"""Resolve Cucumber fixtures once and deduplicate identical CI configurations.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + + +LANES = [("8.11.1", 1, "^TestPublicCucumberTestdrive/passing$"), + ("10.9.0", 4, "^TestPublicCucumberTestdrive$"), + ("13.3.0", 4, "^TestPublicCucumberTestdrive$"), + ("latest", 4, "^TestPublicCucumberTestdrive$")] +DEPENDENCIES = {"nyc": "17.1.0", "c8": "12.0.0", "dd-trace": "6.20.1", + "@istanbuljs/esm-loader-hook": "0.3.0", "tsx": "4.23.15"} +VERSION = re.compile(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?") + + +def execute(command, cwd=None): + return subprocess.run(command, cwd=cwd, check=True, capture_output=True, + text=True, timeout=240).stdout.strip() + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def resolve_version(requested): + version = (requested if VERSION.fullmatch(requested) else + json.loads(execute(["npm", "view", f"@cucumber/cucumber@{requested}", "version", "--json"]))) + if not isinstance(version, str) or not VERSION.fullmatch(version): + raise ValueError(f"Cucumber {requested}: expected one exact version, got {version!r}") + return version + + +def lock_fixture(output, version): + directory = output / version + directory.mkdir() + dependencies = {**DEPENDENCIES, "@cucumber/cucumber": version} + manifest = {"name": "ddtest-cucumber-ci-fixture", "version": "1.0.0", + "private": True, "dependencies": dependencies} + (directory / "package.json").write_text(json.dumps(manifest, indent=2) + "\n") + execute(["npm", "install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund"], + cwd=directory) + lock = json.loads((directory / "package-lock.json").read_text()) + for package, expected in dependencies.items(): + if lock.get("packages", {}).get(f"node_modules/{package}", {}).get("version") != expected: + raise ValueError(f"Cucumber {version}: lockfile did not retain {package}@{expected}") + return fingerprint(lock) + + +def deduplicate(rows): + selected = {} + for row in rows: + # Labels are aliases; every execution input, including the complete + # lockfile, runtime, test filter and shard, participates in the key. + key = fingerprint({name: value for name, value in row.items() if name != "cucumber"}) + if key not in selected: + selected[key] = {**row, "aliases": [row["cucumber"]]} + else: + selected[key]["aliases"].append(row["cucumber"]) + return [{**row, "aliases": ", ".join(row["aliases"])} for row in selected.values()] + + +def build_plan(output): + output.mkdir(parents=True) # Refuse stale files from an earlier resolution. + node, npm = execute(["node", "--version"]).removeprefix("v"), execute(["npm", "--version"]) + if not VERSION.fullmatch(node) or not VERSION.fullmatch(npm): + raise ValueError("expected exact Node and npm versions") + fixtures, rows = {}, [] + for requested, total, tests in LANES: + version = resolve_version(requested) + if version not in fixtures: + fixtures[version] = lock_fixture(output, version) + for shard in range(1, total + 1): + rows.append({"cucumber": requested, "version": version, "fixture": version, + "lock": fixtures[version], "node": node, "npm": npm, + "tests": tests, "shard": f"{shard}/{total}"}) + return {"requested_jobs": len(rows), "matrix": {"include": deduplicate(rows)}} + + +def summary(plan): + rows = plan["matrix"]["include"] + lines = ["## Resolved Cucumber configurations", "", + f"{plan['requested_jobs']} requested lanes → {len(rows)} distinct jobs.", "", + "Matching aliases share a dependency lockfile and execution. Every distinct", + "runtime, dependency graph, test selection and shard still runs.", "", + "| Requested labels | Resolved Cucumber | Shard |", "| --- | --- | --- |"] + lines += [f"| {row['aliases']} | {row['version']} | {row['shard']} |" for row in rows] + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="new directory for fixtures and matrix.json") + parser.add_argument("--github-output", type=Path) + parser.add_argument("--summary", type=Path) + args = parser.parse_args() + try: + plan = build_plan(args.output) + (args.output / "matrix.json").write_text(json.dumps(plan, indent=2) + "\n") + if args.github_output: + with args.github_output.open("a") as output: + output.write("matrix=" + json.dumps(plan["matrix"], separators=(",", ":")) + "\n") + if args.summary: + args.summary.write_text(summary(plan)) + print(summary(plan), end="") + except subprocess.CalledProcessError as error: + parser.exit(1, error.stderr or str(error)) + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + parser.exit(1, str(error) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_smoke.py b/scripts/ci_smoke.py new file mode 100644 index 000000000..90feba8cc --- /dev/null +++ b/scripts/ci_smoke.py @@ -0,0 +1,65 @@ +"""Run a pinned PR integration smoke and require its named cases to pass.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile + +from ci_test_plan import CONFIG + + +def require_cases(passed, required): + missing = set(required) - passed + if missing: + raise ValueError(f"Smoke cases did not pass (missing, skipped or filtered out): {sorted(missing)}") + + +def run_tests(command, env, required): + passed = set() + with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as process: + for line in process.stdout: + try: + event = json.loads(line) + except ValueError: + print(line, end="", flush=True) + continue + if event.get("Action") == "pass" and event.get("Test"): + passed.add(event["Test"]) + if event.get("Output"): + print(event["Output"], end="", flush=True) + if process.wait(): + raise subprocess.CalledProcessError(process.returncode, command) + require_cases(passed, required) + + +def run_smoke(entry): + env = {**os.environ, **entry.get("env", {}), "DDTEST_TEST_SHARD": "1/1"} + with tempfile.TemporaryDirectory(prefix="ddtest-smoke-", dir=os.environ.get("RUNNER_TEMP")) as directory: + if entry.get("dependencies"): + subprocess.run(["npm", "install", "--prefix", directory, "--no-audit", "--no-fund"] + + entry["dependencies"], check=True) + for variable in entry["modules"]: + env[variable] = str(Path(directory) / "node_modules") + for command in entry.get("setup", []): + subprocess.run(command, check=True, env=env) + command = (["xvfb-run", "-a"] if entry.get("xvfb") else []) + [ + "go", "test", "-json", "-count=1", "-timeout=8m", "-run", entry["tests"], + ] + entry.get("packages", ["./internal/testdrive"]) + run_tests(command, env, entry["required"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--id", required=True) + args = parser.parse_args() + config = json.loads(CONFIG.read_text()) + entries = {entry["id"]: entry for suite in config["suites"].values() for entry in suite["smoke"]} + if args.id not in entries: + parser.error(f"unknown smoke: {args.id}") + run_smoke(entries[args.id]) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index 703190e6b..2fbdcb4d4 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -1,114 +1,493 @@ { - "ignored": ["docs/**", "README.md", "AGENTS.md", "ddto/docs/**", "ddto/README.md"], - "shared": ["internal/testdrive/jest*.go"], - "always_run": [ - "test-support", "test-shards", "test", "test-ddtest-instrument-shards", - "test-ddtest-instrument", "lint", "windows-probe-diagnostics", - "onboarding-evaluator-isolation", "integration-plan", "integration-evidence" + "non_product": ["skills/ddtest-onboarding-eval/**"], + "ignored": [ + "docs/**", + "README.md", + "AGENTS.md", + "ddto/docs/**", + "ddto/AGENTS.md", + "ddto/README.md" + ], + "shared": [ + "internal/testdrive/jest*.go" ], "suites": { "python": { - "jobs": ["pytest-compatibility", "testdrive-language-workflow"], + "jobs": [ + "pytest-compatibility", + "testdrive-language-workflow" + ], "inputs": [ - "internal/compatibility/pytest*", "internal/framework/pytest*", - "internal/platform/python*", "internal/platform/scripts/python_*", - "internal/onboard/python*", "internal/onboard/instructions/pytest.md", - "internal/testdrive/python*", "internal/testdrive/pytest*", + "internal/compatibility/pytest*", + "internal/framework/pytest*", + "internal/platform/python*", + "internal/platform/scripts/python_*", + "internal/onboard/python*", + "internal/onboard/instructions/pytest.md", + "internal/testdrive/python*", + "internal/testdrive/pytest*", "internal/testdrive/scripts/pytest_*" + ], + "smoke": [ + { + "id": "python", + "runtime": "python", + "tests": "^TestPublicPythonRubyTestdrives$/^pytest$", + "required": [ + "TestPublicPythonRubyTestdrives/pytest" + ], + "env": { + "DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST": "1" + } + } ] }, "ruby": { - "jobs": ["rspec-compatibility", "minitest-compatibility", "ruby-testdrive"], + "jobs": [ + "rspec-compatibility", + "minitest-compatibility", + "ruby-testdrive" + ], "inputs": [ - "internal/compatibility/rspec*", "internal/compatibility/minitest*", - "internal/framework/rspec*", "internal/framework/minitest*", - "internal/platform/ruby*", "internal/platform/scripts/ruby_*", - "internal/onboard/ruby*", "internal/onboard/instructions/ruby*", + "internal/compatibility/rspec*", + "internal/compatibility/minitest*", + "internal/framework/rspec*", + "internal/framework/minitest*", + "internal/platform/ruby*", + "internal/platform/scripts/ruby_*", + "internal/onboard/ruby*", + "internal/onboard/instructions/ruby*", "internal/onboard/python_ruby_parity_test.go", - "internal/testdrive/ruby*", "internal/testdrive/scripts/ruby_*", + "internal/testdrive/ruby*", + "internal/testdrive/scripts/ruby_*", "internal/testdrive/python_ruby_parity_test.go" + ], + "smoke": [ + { + "id": "ruby", + "runtime": "ruby", + "tests": "^TestRubyValidationIntegration$/^(rspec|minitest)$", + "required": [ + "TestRubyValidationIntegration/rspec", + "TestRubyValidationIntegration/minitest" + ], + "env": { + "DDTEST_RUN_RUBY_INTEGRATION_TEST": "1" + }, + "setup": [ + [ + "gem", + "install", + "datadog-ci", + "--version", + "1.40.0", + "--no-document" + ], + [ + "gem", + "install", + "rspec", + "--version", + "3.13.2", + "--no-document" + ], + [ + "gem", + "install", + "minitest", + "--version", + "6.0.6", + "--no-document" + ], + [ + "gem", + "install", + "rake", + "--version", + "13.4.2", + "--no-document" + ] + ] + } ] }, "java": { - "jobs": ["java-testdrive"], + "jobs": [ + "java-testdrive" + ], "inputs": [ - "internal/framework/jvm*", "internal/platform/java*", "internal/gradle/**", - "internal/maven/**", "internal/onboard/java*", "internal/onboard/instructions/java*", + "internal/framework/jvm*", + "internal/platform/java*", + "internal/gradle/**", + "internal/maven/**", + "internal/onboard/java*", + "internal/onboard/instructions/java*", "internal/testdrive/java*" + ], + "smoke": [ + { + "id": "java-maven", + "runtime": "java", + "tests": "^TestJavaValidationIntegration$/^junit4$", + "required": [ + "TestJavaValidationIntegration/junit4" + ], + "env": { + "DDTEST_RUN_JAVA_INTEGRATION_TEST": "1", + "DDTEST_JAVA_TRACER_VERSION": "1.67.1", + "MAVEN_OPTS": "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } + }, + { + "id": "java-gradle", + "runtime": "gradle", + "tests": "^TestJavaValidationIntegration$/^gradle_junit4$", + "required": [ + "TestJavaValidationIntegration/gradle_junit4" + ], + "env": { + "DDTEST_RUN_JAVA_INTEGRATION_TEST": "1", + "DDTEST_JAVA_TRACER_VERSION": "1.67.1", + "MAVEN_OPTS": "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } + } ] }, "javascript": { - "jobs": ["javascript-platform-compatibility"], + "jobs": [ + "javascript-platform-compatibility" + ], "inputs": [ - "internal/platform/javascript*", "internal/platform/node_options*", - "internal/platform/scripts/javascript*", "internal/framework/javascript*", - "internal/onboard/javascript*", "internal/onboard/node_*" + "internal/platform/javascript*", + "internal/platform/node_options*", + "internal/platform/scripts/javascript*", + "internal/framework/javascript*", + "internal/onboard/javascript*", + "internal/onboard/node_*" + ], + "smoke": [ + { + "id": "javascript", + "runtime": "node", + "packages": [ + "./internal/compatibility" + ], + "tests": "^TestJavaScript(Platform|ActionPreload)Integration$", + "required": [ + "TestJavaScriptPlatformIntegration", + "TestJavaScriptActionPreloadIntegration" + ], + "dependencies": [ + "dd-trace@6.15.0" + ], + "modules": [ + "DDTEST_DD_TRACE_NODE_MODULES" + ] + } ] }, "jest": { - "jobs": ["jest-compatibility", "jest-testdrive-workflow"], - "depends_on": ["javascript"], - "inputs": ["internal/compatibility/jest*", "internal/framework/jest*", "internal/onboard/jest*", "internal/testdrive/jest*"] + "jobs": [ + "jest-compatibility", + "jest-testdrive-workflow" + ], + "depends_on": [ + "javascript" + ], + "inputs": [ + "internal/compatibility/jest*", + "internal/framework/jest*", + "internal/onboard/jest*", + "internal/testdrive/jest*" + ], + "smoke": [ + { + "id": "jest", + "runtime": "node", + "tests": "^TestPublicJestTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicJestTestdrive/passing", + "TestPublicJestTestdrive/existing_failure" + ], + "dependencies": [ + "jest@30.5.1", + "dd-trace@6.15.0", + "typescript@5" + ], + "modules": [ + "DDTEST_JEST_NODE_MODULES" + ] + } + ] }, "vitest": { - "jobs": ["vitest-compatibility", "testdrive-vitest-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "vitest-compatibility", + "testdrive-vitest-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/compatibility/vitest*", "internal/framework/vitest*", "internal/framework/scripts/vitest*", - "internal/onboard/vitest*", "internal/onboard/instructions/vitest.md", - "internal/testdrive/vitest*", "internal/testdrive/scripts/vitest*" + "internal/compatibility/vitest*", + "internal/framework/vitest*", + "internal/framework/scripts/vitest*", + "internal/onboard/vitest*", + "internal/onboard/instructions/vitest.md", + "internal/testdrive/vitest*", + "internal/testdrive/scripts/vitest*" + ], + "smoke": [ + { + "id": "vitest", + "runtime": "node", + "tests": "^TestPublicVitestRunnerTestdrive$/^mixed_CI_paths$", + "required": [ + "TestPublicVitestRunnerTestdrive/mixed_CI_paths" + ], + "dependencies": [ + "vitest@4.1.11", + "@vitest/coverage-v8@4.1.11", + "dd-trace@6.21.0" + ], + "modules": [ + "DDTEST_VITEST_NODE_MODULES" + ], + "env": { + "DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST": "1" + } + } ] }, "mocha": { - "jobs": ["mocha-compatibility", "testdrive-mocha-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "mocha-compatibility", + "testdrive-mocha-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/compatibility/mocha*", "internal/framework/mocha*", "internal/framework/scripts/mocha*", - "internal/onboard/mocha*", "internal/onboard/instructions/mocha.md", - "internal/testdrive/mocha*", "internal/testdrive/scripts/mocha*" + "internal/compatibility/mocha*", + "internal/framework/mocha*", + "internal/framework/scripts/mocha*", + "internal/onboard/mocha*", + "internal/onboard/instructions/mocha.md", + "internal/testdrive/mocha*", + "internal/testdrive/scripts/mocha*" + ], + "smoke": [ + { + "id": "mocha", + "runtime": "node", + "tests": "^TestPublicMochaTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicMochaTestdrive/passing", + "TestPublicMochaTestdrive/existing_failure" + ], + "dependencies": [ + "mocha@11.7.6", + "nyc@17.1.0", + "c8@12.0.0", + "dd-trace@6.21.0" + ], + "modules": [ + "DDTEST_MOCHA_NODE_MODULES" + ] + } ] }, "cypress": { - "jobs": ["cypress-compatibility", "testdrive-cypress-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "cypress-compatibility", + "testdrive-cypress-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/compatibility/cypress*", "internal/framework/cypress*", "internal/framework/scripts/cypress*", - "internal/onboard/cypress*", "internal/onboard/instructions/cypress.md", - "internal/testdrive/cypress*", "internal/testdrive/scripts/cypress*" + "internal/compatibility/cypress*", + "internal/framework/cypress*", + "internal/framework/scripts/cypress*", + "internal/onboard/cypress*", + "internal/onboard/instructions/cypress.md", + "internal/testdrive/cypress*", + "internal/testdrive/scripts/cypress*" + ], + "smoke": [ + { + "id": "cypress", + "runtime": "node", + "tests": "^TestPublicCypressTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicCypressTestdrive/passing", + "TestPublicCypressTestdrive/existing_failure" + ], + "dependencies": [ + "cypress@15.1.0", + "dd-trace@6.20.1", + "typescript@5.9.3" + ], + "modules": [ + "DDTEST_CYPRESS_NODE_MODULES" + ], + "xvfb": true + } ] }, "playwright": { - "jobs": ["playwright-compatibility", "playwright-latest", "testdrive-playwright-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "playwright-compatibility", + "playwright-latest", + "testdrive-playwright-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/compatibility/playwright*", "internal/framework/playwright*", "internal/framework/scripts/playwright*", - "internal/onboard/playwright*", "internal/onboard/instructions/playwright.md", - "internal/testdrive/playwright*", "internal/testdrive/scripts/playwright*" + "internal/compatibility/playwright*", + "internal/framework/playwright*", + "internal/framework/scripts/playwright*", + "internal/onboard/playwright*", + "internal/onboard/instructions/playwright.md", + "internal/testdrive/playwright*", + "internal/testdrive/scripts/playwright*" + ], + "smoke": [ + { + "id": "playwright", + "runtime": "playwright", + "tests": "^TestPublicPlaywrightTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicPlaywrightTestdrive/passing", + "TestPublicPlaywrightTestdrive/existing_failure" + ], + "dependencies": [ + "@playwright/test@1.55.1", + "dd-trace@6.20.1" + ], + "modules": [ + "DDTEST_PLAYWRIGHT_NODE_MODULES" + ] + } ] }, "cucumber": { - "jobs": ["cucumber-compatibility", "testdrive-cucumber-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "cucumber-compatibility", + "cucumber-matrix", + "testdrive-cucumber-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/compatibility/cucumber*", "internal/framework/cucumber*", - "internal/onboard/cucumber*", "internal/onboard/instructions/cucumber.md", - "internal/testdrive/cucumber*", "internal/testdrive/scripts/cucumber*" + "internal/compatibility/cucumber*", + "internal/framework/cucumber*", + "internal/onboard/cucumber*", + "internal/onboard/instructions/cucumber.md", + "internal/testdrive/cucumber*", + "internal/testdrive/scripts/cucumber*" + ], + "smoke": [ + { + "id": "cucumber", + "runtime": "node", + "tests": "^TestPublicCucumberTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicCucumberTestdrive/passing", + "TestPublicCucumberTestdrive/existing_failure" + ], + "dependencies": [ + "@cucumber/cucumber@13.3.0", + "nyc@17.1.0", + "c8@12.0.0", + "dd-trace@6.20.1", + "@istanbuljs/esm-loader-hook@0.3.0", + "tsx@4.23.15" + ], + "modules": [ + "DDTEST_CUCUMBER_NODE_MODULES" + ] + } ] }, "webdriverio": { - "jobs": ["testdrive-webdriverio-workflow"], - "depends_on": ["javascript"], + "jobs": [ + "testdrive-webdriverio-workflow" + ], + "depends_on": [ + "javascript" + ], "inputs": [ - "internal/framework/webdriverio*", "internal/framework/scripts/webdriverio*", "internal/platform/webdriverio*", - "internal/onboard/webdriverio*", "internal/onboard/instructions/webdriverio.md", + "internal/framework/webdriverio*", + "internal/framework/scripts/webdriverio*", + "internal/platform/webdriverio*", + "internal/onboard/webdriverio*", + "internal/onboard/instructions/webdriverio.md", "internal/testdrive/webdriverio*" + ], + "smoke": [ + { + "id": "webdriverio", + "runtime": "node", + "tests": "^TestPublicWebdriverIOTestdrive$/^(passing|existing_failure)$", + "required": [ + "TestPublicWebdriverIOTestdrive/passing", + "TestPublicWebdriverIOTestdrive/existing_failure" + ], + "dependencies": [ + "@wdio/cli@9.20.0", + "@wdio/local-runner@9.20.0", + "@wdio/mocha-framework@9.20.0", + "@wdio/jasmine-framework@9.20.0", + "@wdio/spec-reporter@9.20.0", + "dd-trace@6.20.1" + ], + "modules": [ + "DDTEST_WEBDRIVERIO_NODE_MODULES" + ], + "env": { + "DDTEST_CHROME_BINARY": "/usr/bin/google-chrome" + }, + "xvfb": true + } ] }, "onboarding": { - "jobs": ["onboard-apply-workflow"], - "depends_on": ["javascript", "vitest", "mocha", "cucumber", "webdriverio"], - "inputs": ["internal/testdrive/apply*"], - "additional_inputs": ["internal/onboard/**"] + "jobs": [ + "onboard-apply-workflow" + ], + "depends_on": [ + "javascript", + "vitest", + "mocha", + "cucumber", + "webdriverio" + ], + "inputs": [ + "internal/testdrive/apply*" + ], + "additional_inputs": [ + "internal/onboard/**" + ], + "smoke": [ + { + "id": "onboarding", + "runtime": "node", + "tests": "^TestPublicOnboardApply$", + "required": [ + "TestPublicOnboardApply" + ], + "dependencies": [ + "vitest@4.1.11", + "dd-trace@6.20.1" + ], + "modules": [ + "DDTEST_VITEST_NODE_MODULES" + ] + } + ] } } } diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py index 0980b5784..fc7769364 100644 --- a/scripts/ci_test_plan.py +++ b/scripts/ci_test_plan.py @@ -1,40 +1,26 @@ -"""Shadow integration selection: compare source inputs with a passed main run. - -This never skips tests. Matching sources still need matching resolved dependencies -and runtime identities before their test results could be reused. -""" +"""Select PR smoke checks and affected Go packages from the actual Git diff.""" import argparse from fnmatch import fnmatchcase -import hashlib import json import os from pathlib import Path -import stat import subprocess -import tempfile -SCHEMA = 1 -ARTIFACT = "ci-integration-source-evidence-v1" CONFIG = Path(__file__).with_suffix(".json") -def digest(value): - return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - - def affected_suites(path, config): if any(fnmatchcase(path, pattern) for pattern in config["ignored"]): return set() suites = config["suites"] - if any(fnmatchcase(path, pattern) for pattern in config["shared"]): + if not path.endswith("_test.go") and any(fnmatchcase(path, pattern) for pattern in config["shared"]): return set(suites) affected = {name for name, suite in suites.items() if any(fnmatchcase(path, pattern) for pattern in suite["inputs"])} if not affected: - return set(suites) # Shared and unclassified inputs invalidate everything. - # Additional consumers cannot turn an otherwise shared path into a private one. + return set(suites) affected.update(name for name, suite in suites.items() if any(fnmatchcase(path, pattern) for pattern in suite.get("additional_inputs", []))) while True: @@ -45,142 +31,136 @@ def affected_suites(path, config): affected = expanded -def source_inputs(root, config): - result = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], - cwd=root, check=True, capture_output=True) - inputs = {} - for name in sorted(set(os.fsdecode(result.stdout).split("\0")) - {""}): - if not affected_suites(name, config): - continue - path = root / name - try: - mode = path.lstat().st_mode - except FileNotFoundError: - continue # A tracked deletion must disappear from the fingerprint. - if stat.S_ISLNK(mode): - content, kind = os.fsencode(os.readlink(path)), "symlink" - elif stat.S_ISREG(mode): - content, kind = path.read_bytes(), "executable" if mode & 0o111 else "file" - else: - raise ValueError(f"cannot fingerprint {name}: expected a file or symlink") - inputs[name] = [kind, hashlib.sha256(content).hexdigest()] - return inputs - - -def valid_baseline(baseline, repository): - return (isinstance(baseline, dict) and baseline.get("schema") == SCHEMA - and baseline.get("repository") == repository - and isinstance(baseline.get("head"), str) and isinstance(baseline.get("run_id"), str) - and isinstance(baseline.get("inputs"), dict) - and all(isinstance(path, str) and isinstance(value, list) and len(value) == 2 - and value[0] in ("file", "executable", "symlink") - and isinstance(value[1], str) and len(value[1]) == 64 - for path, value in baseline["inputs"].items())) - - -def github_baseline(repository): - """Only completed, successful pushes to main in this workflow are eligible.""" +def relevant_files(files, config): + if files is None: + return None + ignored = config["ignored"] + config.get("non_product", []) + return [path for path in files if not any(fnmatchcase(path, pattern) for pattern in ignored)] + + +def changed_files(root, base): + if not base or base == "0" * 40: + return None try: - result = subprocess.run([ - "gh", "run", "list", "--repo", repository, "--workflow", "ci.yml", - "--branch", "main", "--event", "push", "--status", "success", "--limit", "10", - "--json", "databaseId,headSha", - ], check=True, capture_output=True, text=True, timeout=30) - runs = json.loads(result.stdout) - for run in runs: - with tempfile.TemporaryDirectory(prefix="ci-test-baseline-") as directory: - result = subprocess.run([ - "gh", "run", "download", str(run["databaseId"]), "--repo", repository, - "--name", ARTIFACT, "--dir", directory, - ], capture_output=True, timeout=30) - if result.returncode: - continue # Old runs and expired artifacts need no special handling. - try: - baseline = json.loads((Path(directory) / "plan.json").read_text()) - except (OSError, ValueError): - continue - if (valid_baseline(baseline, repository) - and baseline.get("head") == run["headSha"] - and baseline.get("run_id") == str(run["databaseId"])): - return baseline - except (OSError, ValueError, KeyError, subprocess.SubprocessError): - pass # An unavailable baseline can never produce an unchanged-source decision. - return None - - -def make_plan(inputs, config, repository, head, baseline=None): - if not valid_baseline(baseline, repository): - baseline = None - previous = baseline["inputs"] if baseline else {} - changed = sorted(path for path in inputs.keys() | previous.keys() - if inputs.get(path) != previous.get(path)) if baseline else [] - affected = {name: [] for name in config["suites"]} - for path in changed: - for name in affected_suites(path, config): - affected[name].append(path) - suites = {} - for name, suite in config["suites"].items(): - selected = {path: value for path, value in inputs.items() - if name in affected_suites(path, config)} - suites[name] = { - "jobs": suite["jobs"], "fingerprint": digest(selected), - "decision": "source_unchanged" if baseline and not affected[name] else "run", - "reason": ("source inputs unchanged; runtime/dependency verification still required" - if baseline and not affected[name] else - "source inputs changed" if baseline else "no eligible baseline"), - "changed_inputs": affected[name], - } + ancestor = subprocess.run(["git", "merge-base", base, "HEAD"], cwd=root, + check=True, capture_output=True, text=True).stdout.strip() + tracked = subprocess.run(["git", "diff", "--no-renames", "--name-only", "-z", ancestor], + cwd=root, check=True, capture_output=True).stdout + added = subprocess.run(["git", "ls-files", "--others", "--exclude-standard", "-z"], + cwd=root, check=True, capture_output=True).stdout + return sorted(set(os.fsdecode(tracked + added).split("\0")) - {""}) + except subprocess.CalledProcessError: + return None + + +def make_plan(files, config): + selected = set(config["suites"]) if files is None else set() + relevant = relevant_files(files, config) + for path in relevant or []: + # The instrumentation is a separate module. Its units and the release + # binary smoke cover changes there; they do not affect framework runners. + if path.startswith("ddto/"): + continue + if path.endswith("_test.go") and not path.endswith("integration_test.go") and not path.startswith("internal/compatibility/"): + continue + selected.update(affected_suites(path, config)) + checks = [{"id": entry["id"], "runtime": entry["runtime"]} + for name, suite in config["suites"].items() if name in selected + for entry in suite["smoke"]] return { - "schema": SCHEMA, "mode": "shadow", "repository": repository, "head": head, - "run_id": os.environ.get("GITHUB_RUN_ID", ""), - "baseline": {key: baseline[key] for key in ("head", "run_id")} if baseline else None, - "external_inputs_verified": False, "always_run": config["always_run"], - "inputs": inputs, "suites": suites, + "mode": "fast", "changed_files": files, + "go": files is None or bool(relevant), + "ddto": files is None or any(path.startswith("ddto/") or path in ( + "main.go", "go.mod", "go.sum", "internal/cmd/instrument.go", + ".github/workflows/go-instrumentation.yml") for path in relevant), + "evaluator": files is None or any(path.startswith("skills/ddtest-onboarding-eval/") + for path in files), + "suites": sorted(selected), + "integrations": [entry for entry in checks if entry["runtime"] != "playwright"], + "playwright": any(entry["runtime"] == "playwright" for entry in checks), } +def json_objects(text): + decoder = json.JSONDecoder() + while text.strip(): + value, end = decoder.raw_decode(text.lstrip()) + yield value + text = text.lstrip()[end:] + + +def select_packages(packages, files, root): + by_dir = {Path(package["Dir"]).resolve(): package["ImportPath"] for package in packages} + all_packages = set(by_dir.values()) + if files is None: + return sorted(all_packages) + direct, production = set(), set() + for path in files: + if not path.endswith(".go") or path.startswith("ddto/"): + return sorted(all_packages) + package = by_dir.get((root / path).resolve().parent) + if package is None: + return sorted(all_packages) + direct.add(package) + if not path.endswith("_test.go"): + production.add(package) + while True: + expanded = production | {package["ImportPath"] for package in packages + if production.intersection(package.get("Imports", []) + + package.get("TestImports", []) + + package.get("XTestImports", []))} + if expanded == production: + return sorted(direct | production) + production = expanded + + +def run_unit_tests(root, files, config, race=False): + relevant = relevant_files(files, config) + if relevant == []: + print("No code inputs changed; no Go packages selected.") + return + command = ["go", "list", "-json"] + (["-race"] if race else []) + ["./..."] + result = subprocess.run(command, cwd=root, check=True, capture_output=True, text=True) + packages = select_packages(list(json_objects(result.stdout)), relevant, root) + if not packages: + raise ValueError("Go reported no packages for changed code") + print(f"Testing {len(packages)} affected Go packages", flush=True) + subprocess.run(["go", "test", "-timeout=10m"] + (["-race"] if race else []) + packages, + cwd=root, check=True) + + def summary(plan): - lines = ["## Integration selection (shadow)", "", - "All tests still run. Unchanged sources are candidates only: resolved dependencies", - "and runtime identities have not been verified. No test evidence is reused.", ""] - if plan["baseline"]: - origin = (f"successful main run {plan['baseline']['run_id']}" - if plan["baseline"]["run_id"] else "local snapshot") - lines.append(f"Compared with {origin} at `{plan['baseline']['head']}`.") - else: - lines.append("No eligible successful main baseline; every suite must run.") - lines += ["", "| Suite | Source decision | Changed inputs |", "| --- | --- | --- |"] - for name, suite in plan["suites"].items(): - paths = suite["changed_inputs"] - details = ", ".join("`" + path.replace("`", "").replace("|", "|") + "`" - for path in paths[:5]) - if len(paths) > 5: - details += f" (+{len(paths) - 5} more)" - lines.append(f"| {name} | {suite['decision']} | {details or '—'} |") - return "\n".join(lines) + "\n" + suites = ", ".join(plan["suites"]) or "none (documentation only)" + return ("## PR test selection\n\n" + f"Selected integration suites: {suites}.\n\n" + f"Native Go checks: {plan['go']}; instrumentation units: {plan['ddto']}.\n\n" + "Historical versions, full scenario matrices and platform parity gate releases " + "through the Full compatibility workflow.\n") def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["plan", "run"]) parser.add_argument("--root", type=Path, default=Path(".")) - parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "local")) - baseline_args = parser.add_mutually_exclusive_group() - baseline_args.add_argument("--baseline", type=Path) - baseline_args.add_argument("--github-baseline", action="store_true") - parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--base", default="origin/main") + parser.add_argument("--race", action="store_true") + parser.add_argument("--output", type=Path) + parser.add_argument("--github-output", type=Path) parser.add_argument("--summary", type=Path) args = parser.parse_args() config = json.loads(CONFIG.read_text()) - baseline = github_baseline(args.repository) if args.github_baseline else None - if args.baseline: - try: - baseline = json.loads(args.baseline.read_text()) - except (OSError, ValueError): - baseline = None - head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=args.root, - check=True, capture_output=True, text=True).stdout.strip() - plan = make_plan(source_inputs(args.root, config), config, args.repository, head, baseline) - args.output.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") + root = args.root.resolve() + files = changed_files(root, args.base) + if args.action == "run": + run_unit_tests(root, files, config, args.race) + return + plan = make_plan(files, config) + if args.output: + args.output.write_text(json.dumps(plan, indent=2) + "\n") + if args.github_output: + with args.github_output.open("a") as output: + for key in ("go", "ddto", "evaluator", "integrations", "playwright"): + output.write(key + "=" + json.dumps(plan[key], separators=(",", ":")) + "\n") if args.summary: args.summary.write_text(summary(plan)) print(summary(plan), end="") diff --git a/scripts/test_ci_cucumber_matrix.py b/scripts/test_ci_cucumber_matrix.py new file mode 100644 index 000000000..5374739b4 --- /dev/null +++ b/scripts/test_ci_cucumber_matrix.py @@ -0,0 +1,154 @@ +"""Keep every distinct configuration while eliminating resolved aliases.""" + +import contextlib +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import ci_cucumber_matrix as planner + + +class MatrixTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.output = Path(directory.name) / "fixtures" + self.latest = "13.3.0" + self.commands = [] + + def execute(self, command, cwd=None): + self.commands.append(command) + if command == ["node", "--version"]: + return "v24.14.1" + if command == ["npm", "--version"]: + return "11.11.0" + if command[:2] == ["npm", "view"]: + return json.dumps(self.latest) + self.assertEqual(command, ["npm", "install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund"]) + manifest = json.loads((cwd / "package.json").read_text()) + lock = {"lockfileVersion": 3, "packages": { + f"node_modules/{name}": {"version": version} + for name, version in manifest["dependencies"].items()}} + lock["packages"]["node_modules/transitive"] = {"version": "1.2.3", "integrity": "sha512-fixture"} + (cwd / "package-lock.json").write_text(json.dumps(lock)) + return "" + + def build(self): + with patch.object(planner, "execute", side_effect=self.execute): + return planner.build_plan(self.output) + + def test_equal_latest_and_pin_share_locks_and_jobs_but_keep_every_shard(self): + plan = self.build() + rows = plan["matrix"]["include"] + self.assertEqual(plan["requested_jobs"], 13) + self.assertEqual(len(rows), 9) + self.assertEqual(sum(len(row["aliases"].split(", ")) for row in rows), 13) + shared = [row for row in rows if row["version"] == "13.3.0"] + self.assertEqual({row["shard"] for row in shared}, {"1/4", "2/4", "3/4", "4/4"}) + self.assertTrue(all(row["aliases"] == "13.3.0, latest" for row in shared)) + self.assertEqual({path.name for path in self.output.iterdir()}, {"8.11.1", "10.9.0", "13.3.0"}) + self.assertEqual(sum(command[:2] == ["npm", "view"] for command in self.commands), 1) + for row in rows: + lock = json.loads((self.output / row["fixture"] / "package-lock.json").read_text()) + self.assertEqual(row["lock"], planner.fingerprint(lock)) + self.assertEqual((row["node"], row["npm"]), ("24.14.1", "11.11.0")) + + def test_a_new_latest_release_retains_all_thirteen_jobs(self): + self.latest = "13.4.0" + rows = self.build()["matrix"]["include"] + self.assertEqual(len(rows), 13) + self.assertEqual(len([row for row in rows if row["version"] == "13.4.0"]), 4) + self.assertTrue(all(row["aliases"] == row["cucumber"] for row in rows)) + + def test_version_equality_never_collapses_floor_and_full_scenarios(self): + self.latest = "8.11.1" + rows = self.build()["matrix"]["include"] + self.assertEqual(len(rows), 13) + floor = [row for row in rows if row["cucumber"] == "8.11.1"] + latest = [row for row in rows if row["cucumber"] == "latest"] + self.assertEqual(floor[0]["tests"], "^TestPublicCucumberTestdrive/passing$") + self.assertTrue(all(row["tests"] == "^TestPublicCucumberTestdrive$" for row in latest)) + + def test_runtime_graph_filter_and_shard_differences_cannot_be_deduplicated(self): + row = self.build()["matrix"]["include"][5] + row = {key: value for key, value in row.items() if key != "aliases"} + alias = {**row, "cucumber": "latest"} + self.assertEqual(len(planner.deduplicate([row, alias])), 1) + for key in ("node", "npm", "lock", "tests", "shard"): + with self.subTest(key=key): + self.assertEqual(len(planner.deduplicate([row, {**alias, key: "different"}])), 2) + lock = json.loads((self.output / row["fixture"] / "package-lock.json").read_text()) + lock["packages"]["node_modules/transitive"]["version"] = "1.2.4" + self.assertNotEqual(planner.fingerprint(lock), row["lock"]) + + def test_ambiguous_or_invalid_registry_versions_fail(self): + for version in (["13.3.0", "13.4.0"], None, "latest", "../unsafe"): + self.latest = version + with self.subTest(version=version), patch.object(planner, "execute", side_effect=self.execute): + with self.assertRaises(ValueError): + planner.resolve_version("latest") + + def test_lock_resolution_failure_never_publishes_a_partial_matrix(self): + github_output = self.output.parent / "github-output" + command = ["planner", "--output", str(self.output), "--github-output", str(github_output)] + with patch.object(sys, "argv", command), patch.object(planner, "execute", side_effect=[ + "v24.14.1", "11.11.0", subprocess.CalledProcessError(1, ["npm"], stderr="registry unavailable")]): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + planner.main() + self.assertEqual(error.exception.code, 1) + self.assertFalse(github_output.exists()) + self.assertFalse((self.output / "matrix.json").exists()) + + def test_success_publishes_the_same_complete_matrix_to_github_and_the_artifact(self): + github_output = self.output.parent / "github-output" + github_output.write_text("existing=value\n") + summary = self.output.parent / "summary.md" + command = ["planner", "--output", str(self.output), "--github-output", str(github_output), + "--summary", str(summary)] + with patch.object(sys, "argv", command), patch.object(planner, "execute", side_effect=self.execute): + with contextlib.redirect_stdout(io.StringIO()): + planner.main() + lines = github_output.read_text().splitlines() + self.assertEqual(lines[0], "existing=value") + self.assertEqual(len(lines), 2) + key, value = lines[1].split("=", 1) + self.assertEqual(key, "matrix") + artifact = json.loads((self.output / "matrix.json").read_text()) + self.assertEqual(json.loads(value), artifact["matrix"]) + self.assertEqual(len(artifact["matrix"]["include"]), 9) + self.assertIn("13 requested lanes → 9 distinct jobs", summary.read_text()) + + def test_an_incomplete_or_wrong_lockfile_fails(self): + self.output.mkdir() + with patch.object(planner, "execute") as execute: + execute.side_effect = lambda command, cwd: (cwd / "package-lock.json").write_text('{"packages":{}}') + with self.assertRaisesRegex(ValueError, "lockfile did not retain"): + planner.lock_fixture(self.output, "13.3.0") + + def test_existing_output_is_not_reused(self): + self.build() + with self.assertRaises(FileExistsError): + self.build() + + +class WorkflowTests(unittest.TestCase): + def test_workers_use_the_planned_runtime_and_lockfile_without_resolving_again(self): + workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/ci-full.yml").read_text() + worker = workflow.split(" testdrive-cucumber-workflow:\n", 1)[1].split("\n testdrive-webdriverio-workflow:", 1)[0] + self.assertIn("configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }}", worker) + self.assertIn("node-version: ${{ matrix.configuration.node }}", worker) + self.assertIn('test "$(npm --version)" =', worker) + self.assertIn("name: ci-cucumber-fixtures", worker) + self.assertIn('npm ci --prefix "$RUNNER_TEMP/cucumber-testdrive"', worker) + self.assertNotIn("npm install", worker) + self.assertNotIn("npm view", worker) + self.assertIn("-run '${{ matrix.configuration.tests }}'", worker) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_ci_smoke.py b/scripts/test_ci_smoke.py new file mode 100644 index 000000000..0e56ef350 --- /dev/null +++ b/scripts/test_ci_smoke.py @@ -0,0 +1,35 @@ +"""A selected smoke must execute its named cases, not pass after filtering them.""" + +import io +import subprocess +import unittest +from unittest.mock import patch + +import ci_smoke + + +class SmokeTests(unittest.TestCase): + def test_missing_skipped_or_filtered_cases_cannot_pass(self): + for passed in (set(), {"TestSmoke"}, {"TestSmoke/other"}): + with self.assertRaises(ValueError): + ci_smoke.require_cases(passed, ["TestSmoke/passing"]) + ci_smoke.require_cases({"TestSmoke/passing"}, ["TestSmoke/passing"]) + + def test_json_exit_and_required_case_both_gate_success(self): + class Process: + returncode = 0 + stdout = io.StringIO('{"Action":"pass","Test":"TestSmoke/passing"}\n') + def __enter__(self): return self + def __exit__(self, *args): pass + def wait(self): return self.returncode + with patch.object(ci_smoke.subprocess, "Popen", return_value=Process()): + ci_smoke.run_tests(["go"], {}, ["TestSmoke/passing"]) + process = Process() + process.returncode = 1 + with patch.object(ci_smoke.subprocess, "Popen", return_value=process): + with self.assertRaises(subprocess.CalledProcessError): + ci_smoke.run_tests(["go"], {}, ["TestSmoke/passing"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index cd5621d6c..b7051440a 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -1,6 +1,6 @@ -"""Exercise selective CI decisions without using a live GitHub account.""" +"""Verify active PR selection, reverse Go dependencies and required check gates.""" -import hashlib +import copy import json import os from pathlib import Path @@ -17,185 +17,172 @@ CONFIG = json.loads(planner.CONFIG.read_text()) -def source(content, kind="file"): - return [kind, hashlib.sha256(content.encode()).hexdigest()] - - class SelectionTests(unittest.TestCase): - def setUp(self): - self.inputs = {"internal/ext/command.go": source("shared"), - "internal/testdrive/python_validation.go": source("python"), - "internal/testdrive/java_validation.go": source("java")} - self.baseline = planner.make_plan(self.inputs, CONFIG, "DataDog/ddtest", "old") - - def plan(self, inputs, baseline=None): - return planner.make_plan(inputs, CONFIG, "DataDog/ddtest", "new", - self.baseline if baseline is None else baseline) - - def running(self, plan): - return {name for name, suite in plan["suites"].items() if suite["decision"] == "run"} - - def test_same_sources_are_candidates_but_never_certify_external_inputs(self): - plan = self.plan(dict(reversed(list(self.inputs.items())))) - self.assertFalse(self.running(plan)) - self.assertEqual(plan["mode"], "shadow") - self.assertFalse(plan["external_inputs_verified"]) - for name in CONFIG["suites"]: - self.assertEqual(plan["suites"][name]["fingerprint"], - self.baseline["suites"][name]["fingerprint"]) - self.assertIn("All tests still run", planner.summary(plan)) - self.assertIn("runtime identities have not been verified", planner.summary(plan)) - - def test_python_edit_keeps_unrelated_suites_as_candidates(self): - inputs = {**self.inputs, "internal/testdrive/python_validation.go": source("changed")} - plan = self.plan(inputs) - self.assertEqual(self.running(plan), {"python"}) - self.assertEqual(plan["suites"]["python"]["changed_inputs"], - ["internal/testdrive/python_validation.go"]) - - def test_shared_unknown_build_and_workflow_edits_invalidate_every_suite(self): - for path in ("internal/ext/command.go", "internal/testdrive/new_shared.go", - "internal/onboard/ci_shell_state.go", "internal/onboard/new_shared.go", - "go.mod", "go.sum", "main.go", "ddto/cli/cli.go", - ".github/workflows/ci.yml", "scripts/ci_test_plan.json", - "internal/testdrive/jest_features.go", "internal/testdrive/jest_preflight.go", - "internal/testdrive/testdata/new fixture/data.json"): + def test_python_change_selects_only_python_smoke(self): + plan = planner.make_plan(["internal/testdrive/python_validation.go"], CONFIG) + self.assertEqual(plan["suites"], ["python"]) + self.assertEqual(plan["integrations"], [{"id": "python", "runtime": "python"}]) + self.assertTrue(plan["go"]) + self.assertFalse(plan["ddto"]) + self.assertEqual(plan["mode"], "fast") + + def test_shared_unknown_build_and_deleted_inputs_select_every_smoke(self): + for path in ("internal/ext/command.go", "internal/onboard/ci_shell_state.go", + "internal/testdrive/jest_features.go", "internal/testdrive/new_fixture.json", + "go.mod", "go.sum", ".github/workflows/ci.yml"): with self.subTest(path=path): - self.assertEqual(self.running(self.plan({**self.inputs, path: source("changed")})), - set(CONFIG["suites"])) - - def test_test_edits_additions_deletions_renames_and_modes_invalidate_the_owner(self): - old = "internal/testdrive/python_validation.go" - for inputs in ({key: value for key, value in self.inputs.items() if key != old}, - {**self.inputs, old: source("python", "executable")}, - {**self.inputs, "internal/testdrive/python_new_test.go": source("new test")}, - {**{key: value for key, value in self.inputs.items() if key != old}, - "internal/testdrive/java_renamed.go": source("python")}): - with self.subTest(inputs=inputs): - self.assertIn("python", self.running(self.plan(inputs))) - renamed = {**{key: value for key, value in self.inputs.items() if key != old}, - "internal/testdrive/java_renamed.go": source("python")} - self.assertEqual(self.running(self.plan(renamed)), {"python", "java"}) - - def test_embedded_scripts_invalidate_their_consumers(self): - for path, owner in (("internal/testdrive/scripts/pytest_observer.py", "python"), - ("internal/platform/scripts/ruby_env.rb", "ruby"), - ("internal/framework/scripts/vitest.mjs", "vitest"), - ("internal/onboard/instructions/pytest.md", "python")): - with self.subTest(path=path): - self.assertIn(owner, self.running(self.plan({**self.inputs, path: source("changed")}))) - - def test_javascript_inputs_propagate_to_all_javascript_suites_and_onboarding(self): - plan = self.plan({**self.inputs, "internal/platform/scripts/javascript_env.js": source("changed")}) - self.assertEqual(self.running(plan), {"javascript", "jest", "vitest", "mocha", "cypress", - "playwright", "cucumber", "webdriverio", "onboarding"}) - - def test_overlapping_and_transitive_inputs_keep_every_consumer(self): - self.assertEqual(planner.affected_suites("internal/testdrive/python_ruby_parity_test.go", CONFIG), - {"python", "ruby"}) - self.assertEqual(planner.affected_suites("internal/onboard/python_venv.go", CONFIG), - {"python", "onboarding"}) - self.assertEqual(planner.affected_suites("internal/framework/scripts/vitest.mjs", CONFIG), - {"vitest", "onboarding"}) - - def test_only_explicit_documentation_paths_are_ignored(self): - for path in ("docs/onboarding.md", "README.md", "AGENTS.md", "ddto/docs/validation.md"): - self.assertFalse(planner.affected_suites(path, CONFIG)) - self.assertTrue(planner.affected_suites("internal/onboard/instructions/pytest.md", CONFIG)) - - def test_missing_foreign_incompatible_or_malformed_baselines_run_everything(self): - invalid = [None, [], {}, {**self.baseline, "repository": "someone/else"}, - {**self.baseline, "schema": 0}, {**self.baseline, "inputs": []}, - {**self.baseline, "inputs": {"go.mod": "malformed"}}] - for baseline in invalid: - with self.subTest(baseline=baseline): - plan = planner.make_plan(self.inputs, CONFIG, "DataDog/ddtest", "new", baseline) - self.assertEqual(self.running(plan), set(CONFIG["suites"])) - self.assertIsNone(plan["baseline"]) - - -class SnapshotTests(unittest.TestCase): - def test_snapshot_uses_content_and_modes_without_following_symlinks(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - file = root / "go.mod" - file.write_text("first") - (root / "link").symlink_to("missing-outside-source") - output = subprocess.CompletedProcess([], 0, b"go.mod\0link\0deleted.go\0") - with patch.object(planner.subprocess, "run", return_value=output): - first = planner.source_inputs(root, CONFIG) - os.utime(file, (1, 1)) - self.assertEqual(first, planner.source_inputs(root, CONFIG)) - file.write_text("other") - self.assertNotEqual(first, planner.source_inputs(root, CONFIG)) - file.chmod(0o755) - self.assertEqual(planner.source_inputs(root, CONFIG)["go.mod"][0], "executable") - self.assertNotIn("deleted.go", first) - self.assertEqual(first["link"], source("missing-outside-source", "symlink")) - - -class BaselineTests(unittest.TestCase): - def test_lookup_pins_workflow_branch_event_success_and_artifact_identity(self): - baseline = planner.make_plan({}, CONFIG, "DataDog/ddtest", "good") - baseline["run_id"] = "2" - - def run(command, **kwargs): - if command[1:3] == ["run", "list"]: - return subprocess.CompletedProcess(command, 0, json.dumps([ - {"databaseId": 1, "headSha": "old"}, {"databaseId": 2, "headSha": "good"}])) - if command[3] == "1": - return subprocess.CompletedProcess(command, 1) - (Path(command[-1]) / "plan.json").write_text(json.dumps(baseline)) - return subprocess.CompletedProcess(command, 0) - - with patch.object(planner.subprocess, "run", side_effect=run) as invoke: - self.assertEqual(planner.github_baseline("DataDog/ddtest"), baseline) - command = invoke.call_args_list[0].args[0] - for flag, value in (("--workflow", "ci.yml"), ("--branch", "main"), - ("--event", "push"), ("--status", "success"), ("--repo", "DataDog/ddtest")): - self.assertEqual(command[command.index(flag) + 1], value) - self.assertIn(planner.ARTIFACT, invoke.call_args_list[-1].args[0]) - - baseline["head"] = "wrong-revision" - with patch.object(planner.subprocess, "run", side_effect=run): - self.assertIsNone(planner.github_baseline("DataDog/ddtest")) - - def test_network_authentication_and_timeout_failures_produce_no_baseline(self): - for error in (OSError("missing gh"), subprocess.CalledProcessError(1, ["gh"]), - subprocess.TimeoutExpired(["gh"], 30)): - with self.subTest(error=error), patch.object(planner.subprocess, "run", side_effect=error): - self.assertIsNone(planner.github_baseline("DataDog/ddtest")) + plan = planner.make_plan([path], CONFIG) + self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) + self.assertTrue(plan["playwright"]) + + def test_docs_only_and_empty_diff_omit_runtime_checks(self): + for files in ([], ["docs/onboarding.md", "README.md", "ddto/docs/validation.md"]): + plan = planner.make_plan(files, CONFIG) + self.assertFalse(plan["go"]) + self.assertFalse(plan["ddto"]) + self.assertFalse(plan["playwright"]) + self.assertEqual(plan["integrations"], []) + + def test_unknown_history_runs_all_checks(self): + plan = planner.make_plan(None, CONFIG) + self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) + self.assertTrue(plan["go"] and plan["ddto"] and plan["evaluator"]) + + def test_embedded_instructions_and_scripts_are_code_inputs(self): + for path in ("internal/onboard/instructions/pytest.md", "internal/testdrive/scripts/pytest_observer.py"): + plan = planner.make_plan([path], CONFIG) + self.assertIn("python", plan["suites"]) + self.assertTrue(plan["go"]) + + def test_javascript_dependencies_propagate_to_every_javascript_consumer(self): + plan = planner.make_plan(["internal/platform/scripts/javascript_env.js"], CONFIG) + self.assertEqual(set(plan["suites"]), {"javascript", "jest", "vitest", "mocha", "cypress", + "playwright", "cucumber", "webdriverio", "onboarding"}) + + def test_instrumentation_changes_enable_native_module_checks(self): + for path in ("ddto/internal/runner/run.go", "main.go", "go.mod", "internal/cmd/instrument.go"): + self.assertTrue(planner.make_plan([path], CONFIG)["ddto"]) + + def test_instrumentation_module_changes_do_not_run_other_language_frameworks(self): + plan = planner.make_plan(["ddto/internal/runner/run.go"], CONFIG) + self.assertTrue(plan["ddto"] and plan["go"]) + self.assertEqual(plan["integrations"], []) + self.assertFalse(plan["playwright"]) + + def test_evaluator_checks_are_selected_for_evaluator_changes(self): + plan = planner.make_plan(["skills/ddtest-onboarding-eval/scripts/run_eval.py"], CONFIG) + self.assertTrue(plan["evaluator"]) + self.assertFalse(plan["go"]) + self.assertEqual(plan["integrations"], []) + self.assertFalse(planner.make_plan(["internal/testdrive/python_validation.go"], CONFIG)["evaluator"]) + + def test_unit_test_edits_run_go_without_external_framework_smokes(self): + plan = planner.make_plan(["internal/testdrive/jest_test.go"], CONFIG) + self.assertTrue(plan["go"]) + self.assertEqual(plan["integrations"], []) + self.assertFalse(plan["playwright"]) + + +class GoSelectionTests(unittest.TestCase): + def setUp(self): + self.root = Path("/example").resolve() + self.packages = [ + {"Dir": str(self.root / "lib"), "ImportPath": "example/lib"}, + {"Dir": str(self.root / "consumer"), "ImportPath": "example/consumer", "Imports": ["example/lib"]}, + {"Dir": str(self.root / "external"), "ImportPath": "example/external", "XTestImports": ["example/consumer"]}, + {"Dir": str(self.root / "other"), "ImportPath": "example/other"}, + ] + + def test_production_change_reaches_transitive_test_importers(self): + self.assertEqual(planner.select_packages(self.packages, ["lib/lib.go"], self.root), + ["example/consumer", "example/external", "example/lib"]) + + def test_test_only_change_does_not_invalidate_production_consumers(self): + self.assertEqual(planner.select_packages(self.packages, ["lib/lib_test.go"], self.root), ["example/lib"]) + + def test_non_go_inputs_deleted_packages_and_unknown_history_run_all_packages(self): + expected = sorted(package["ImportPath"] for package in self.packages) + for files in (None, ["go.mod"], ["lib/testdata/input.txt"], ["deleted/old.go"], ["ddto/runtime.go"]): + self.assertEqual(planner.select_packages(self.packages, files, self.root), expected) + + def test_go_json_stream_is_decoded_without_compiling_test_binaries(self): + text = "\n" + "\n".join(json.dumps(package) for package in self.packages) + self.assertEqual(list(planner.json_objects(text)), self.packages) + + def test_docs_do_not_start_go(self): + with patch.object(planner.subprocess, "run") as run: + planner.run_unit_tests(self.root, ["README.md"], CONFIG) + run.assert_not_called() + + +class GitDiffTests(unittest.TestCase): + def test_diff_includes_worktree_deletions_both_sides_of_renames_and_untracked_files(self): + responses = [subprocess.CompletedProcess([], 0, "base\n"), + subprocess.CompletedProcess([], 0, b"old.go\0new.go\0deleted.go\0"), + subprocess.CompletedProcess([], 0, b"untracked.go\0")] + with patch.object(planner.subprocess, "run", side_effect=responses) as run: + self.assertEqual(planner.changed_files(Path('.'), "origin/main"), + ["deleted.go", "new.go", "old.go", "untracked.go"]) + self.assertIn("--no-renames", run.call_args_list[1].args[0]) + + def test_missing_history_cannot_skip_checks(self): + for base in ("", "0" * 40): + self.assertIsNone(planner.changed_files(Path('.'), base)) + with patch.object(planner.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["git"])): + self.assertIsNone(planner.changed_files(Path('.'), "missing")) class WorkflowTests(unittest.TestCase): - def test_every_job_has_one_owner_and_all_suite_jobs_gate_evidence(self): - workflow = (ROOT / ".github/workflows/ci.yml").read_text() - jobs = re.findall(r"^ ([\w-]+):$", workflow.split("jobs:\n", 1)[1], re.M) - registered = CONFIG["always_run"] + [job for suite in CONFIG["suites"].values() for job in suite["jobs"]] - self.assertCountEqual(jobs, registered) - evidence = workflow.split(" integration-evidence:\n", 1)[1] - dependencies = re.search(r" needs: \[([^]]+)\]", evidence).group(1) - self.assertCountEqual(re.split(r",\s*", dependencies), - ["integration-plan"] + [job for suite in CONFIG["suites"].values() for job in suite["jobs"]]) + def test_full_matrix_is_preserved_and_gates_release(self): + full = (ROOT / ".github/workflows/ci-full.yml").read_text() + jobs = set(re.findall(r"^ ([\w-]+):$", full.split("jobs:\n", 1)[1], re.M)) for suite in CONFIG["suites"].values(): - self.assertTrue(set(suite.get("depends_on", [])).issubset(CONFIG["suites"])) - - def test_shadow_decisions_do_not_gate_existing_jobs(self): - workflow = (ROOT / ".github/workflows/ci.yml").read_text().split("jobs:\n", 1)[1] - for block in re.split(r"(?=^ [\w-]+:\n)", workflow, flags=re.M): - if block.startswith((" integration-plan:", " integration-evidence:")): - continue - self.assertNotIn("integration-plan", block) - self.assertNotIn("source_unchanged", block) - - def test_failed_skipped_or_cancelled_jobs_cannot_publish_evidence(self): - workflow = (ROOT / ".github/workflows/ci.yml").read_text().split(" integration-evidence:\n", 1)[1] + self.assertTrue(set(suite["jobs"]).issubset(jobs)) + self.assertIn("uses: ./.github/workflows/ci.yml", full) + self.assertIn("uses: ./.github/workflows/go-instrumentation.yml", full) + release = (ROOT / ".github/workflows/release.yml").read_text() + self.assertIn("uses: ./.github/workflows/ci-full.yml", release) + self.assertIn("needs: compatibility", release) + + def test_fast_workflow_omits_historical_matrix_coverage_and_sdk_weaving(self): + fast = (ROOT / ".github/workflows/ci.yml").read_text() + self.assertNotIn("test-shards:", fast) + self.assertNotIn("Configure Datadog Test Optimization", fast) + self.assertNotIn("-coverpkg=./...", fast) + self.assertNotIn("shadow", fast) + self.assertIn("ci_test_plan.py run", fast) + self.assertIn("ci_smoke.py --id", fast) + self.assertIn(" lint:\n", fast) + + def test_fast_smokes_have_unique_ids_and_explicit_required_cases(self): + entries = [entry for suite in CONFIG["suites"].values() for entry in suite["smoke"]] + self.assertEqual(len(entries), len({entry["id"] for entry in entries})) + for entry in entries: + self.assertTrue(entry["required"]) + self.assertTrue(entry["tests"].startswith("^Test")) + + def test_required_test_gate_rejects_failed_cancelled_and_unexpectedly_skipped_checks(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text().split(" test:\n", 1)[1] guard = re.search(r"python3 - <<'PY'\n(.*?)\n PY", workflow, re.S).group(1) code = "\n".join(line[10:] for line in guard.splitlines()) - for status in ("success", "failure", "skipped", "cancelled"): - result = subprocess.run(["python3", "-c", code], capture_output=True, - env={**os.environ, "RESULTS": json.dumps({"matrix": {"result": status}})}) - self.assertEqual(result.returncode == 0, status == "success") + outputs = {"go": "true", "ddto": "false", "evaluator": "false", "integrations": "[]", "playwright": "false"} + results = {name: {"result": "success"} for name in ("plan", "scripts", "unit", "instrument", "lint")} + results.update({name: {"result": "skipped"} for name in ("integration", "playwright", "instrumentation-unit", "evaluator-isolation")}) + results["plan"]["outputs"] = outputs + def run(value): + return subprocess.run(["python3", "-c", code], capture_output=True, + env={**os.environ, "RESULTS": json.dumps(value)}).returncode + self.assertEqual(run(results), 0) + for name in ("plan", "scripts", "unit", "instrument", "lint"): + for status in ("failure", "cancelled", "skipped"): + broken = copy.deepcopy(results) + broken[name]["result"] = status + self.assertNotEqual(run(broken), 0, (name, status)) + selected = copy.deepcopy(results) + selected["plan"]["outputs"]["integrations"] = '[{"id":"python"}]' + self.assertNotEqual(run(selected), 0) + selected["integration"]["result"] = "success" + self.assertEqual(run(selected), 0) if __name__ == "__main__": From 19e2f9c80045b798bc543985df38da303e52ba99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 07:45:04 +0100 Subject: [PATCH 03/14] Fix CI input ownership and restore runtime regression checks --- .github/workflows/ci.yml | 11 ++++++++++ docs/ci-test-selection.md | 3 +++ scripts/ci_test_plan.json | 14 ++++++++---- scripts/test_ci_test_plan.py | 42 ++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 423886ff6..63a617157 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,9 +58,20 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: '3.4' + - name: Install runtime-backed unit test prerequisites + run: | + npm install --global yarn@1.22.22 + python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - name: Test affected Go packages and their dependents env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + DDTEST_REQUIRE_PROJECT_RUNTIMES: '1' run: python3 scripts/ci_test_plan.py run --base "$BASE" --race scripts: diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index 634491800..13206017c 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -59,6 +59,9 @@ JavaScript frameworks. Shared or unclassified production inputs select every PR smoke. Instrumentation module changes select its units and the root/release smoke without unrelated framework integrations. Unit-test-only edits omit external smokes; integration-test edits still select their framework. +The native Go job installs Python/pytest/ddtrace, Ruby/Bundler and Node/Yarn, +and sets `DDTEST_REQUIRE_PROJECT_RUNTIMES=1` so focused runtime regressions in +selected packages fail on missing prerequisites instead of silently skipping. Documentation-only changes omit runtime jobs; evaluator-only changes run the evaluator checks. The small Python suite tests CI and installer scripts on every PR. Missing base history selects all PR checks, never an empty plan. diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index 2fbdcb4d4..66a985ba4 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -9,7 +9,9 @@ "ddto/README.md" ], "shared": [ - "internal/testdrive/jest*.go" + "internal/testdrive/jest*.go", + "internal/onboard/jest_command.go", + "internal/onboard/python_handwritten.go" ], "suites": { "python": { @@ -163,7 +165,8 @@ "internal/platform/scripts/javascript*", "internal/framework/javascript*", "internal/onboard/javascript*", - "internal/onboard/node_*" + "internal/onboard/node_*", + "internal/testdrive/javascript*" ], "smoke": [ { @@ -312,7 +315,8 @@ "internal/onboard/cypress*", "internal/onboard/instructions/cypress.md", "internal/testdrive/cypress*", - "internal/testdrive/scripts/cypress*" + "internal/testdrive/scripts/cypress*", + "internal/testdrive/cucumber_coverage.go" ], "smoke": [ { @@ -348,6 +352,7 @@ "internal/compatibility/playwright*", "internal/framework/playwright*", "internal/framework/scripts/playwright*", + "internal/framework/cypress.go", "internal/onboard/playwright*", "internal/onboard/instructions/playwright.md", "internal/testdrive/playwright*", @@ -425,7 +430,8 @@ "internal/platform/webdriverio*", "internal/onboard/webdriverio*", "internal/onboard/instructions/webdriverio.md", - "internal/testdrive/webdriverio*" + "internal/testdrive/webdriverio*", + "internal/testdrive/mocha_validation_test.go" ], "smoke": [ { diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index b7051440a..280b28bf0 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -59,6 +59,37 @@ def test_javascript_dependencies_propagate_to_every_javascript_consumer(self): self.assertEqual(set(plan["suites"]), {"javascript", "jest", "vitest", "mocha", "cypress", "playwright", "cucumber", "webdriverio", "onboarding"}) + def test_javascript_testdrive_sources_select_every_javascript_consumer(self): + consumers = {"javascript", "jest", "vitest", "mocha", "cypress", "playwright", + "cucumber", "webdriverio", "onboarding"} + for path in ("internal/testdrive/javascript.go", "internal/testdrive/javascript_language.go"): + with self.subTest(path=path): + plan = planner.make_plan([path], CONFIG) + self.assertTrue(consumers.issubset(plan["suites"])) + self.assertFalse({"python", "ruby"}.intersection(plan["suites"])) + + def test_cross_framework_onboarding_helpers_select_all_smokes(self): + for path in ("internal/onboard/jest_command.go", "internal/onboard/python_handwritten.go"): + with self.subTest(path=path): + plan = planner.make_plan([path], CONFIG) + self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) + + def test_cross_framework_coverage_and_path_helpers_select_their_consumers(self): + for path, expected in ( + ("internal/testdrive/cucumber_coverage.go", {"cucumber", "cypress", "onboarding"}), + ("internal/framework/cypress.go", {"cypress", "playwright"}), + ): + with self.subTest(path=path): + self.assertEqual(set(planner.make_plan([path], CONFIG)["suites"]), expected) + + def test_mocha_fixture_helper_tracks_webdriverio_without_expanding_unit_only_checks(self): + path = "internal/testdrive/mocha_validation_test.go" + self.assertEqual(planner.affected_suites(path, CONFIG), {"mocha", "webdriverio", "onboarding"}) + plan = planner.make_plan([path], CONFIG) + self.assertTrue(plan["go"]) + self.assertEqual(plan["integrations"], []) + self.assertFalse(plan["playwright"]) + def test_instrumentation_changes_enable_native_module_checks(self): for path in ("ddto/internal/runner/run.go", "main.go", "go.mod", "internal/cmd/instrument.go"): self.assertTrue(planner.make_plan([path], CONFIG)["ddto"]) @@ -161,6 +192,17 @@ def test_fast_smokes_have_unique_ids_and_explicit_required_cases(self): self.assertTrue(entry["required"]) self.assertTrue(entry["tests"].startswith("^Test")) + def test_fast_units_provision_and_require_project_runtimes(self): + fast = (ROOT / ".github/workflows/ci.yml").read_text() + unit = fast.split(" unit:\n", 1)[1].split("\n scripts:\n", 1)[0] + for prerequisite in ("actions/setup-node@", "actions/setup-python@", "ruby/setup-ruby@", + "npm install --global yarn@1.22.22", + 'python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist'): + self.assertIn(prerequisite, unit) + self.assertLess(unit.index(prerequisite), unit.index("ci_test_plan.py run")) + check = unit.split("- name: Test affected Go packages and their dependents", 1)[1] + self.assertIn("DDTEST_REQUIRE_PROJECT_RUNTIMES: '1'", check) + def test_required_test_gate_rejects_failed_cancelled_and_unexpectedly_skipped_checks(self): workflow = (ROOT / ".github/workflows/ci.yml").read_text().split(" test:\n", 1)[1] guard = re.search(r"python3 - <<'PY'\n(.*?)\n PY", workflow, re.S).group(1) From 733d9210d1017f0409ada61657a70278e64a362a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 10:29:49 +0100 Subject: [PATCH 04/14] Restore the full package timeout for affected Go tests --- .github/workflows/ci.yml | 2 +- scripts/ci_test_plan.py | 2 +- scripts/test_ci_test_plan.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63a617157..afadd6fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: needs: plan if: needs.plan.outputs.go == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py index fc7769364..a14ffcbb3 100644 --- a/scripts/ci_test_plan.py +++ b/scripts/ci_test_plan.py @@ -125,7 +125,7 @@ def run_unit_tests(root, files, config, race=False): if not packages: raise ValueError("Go reported no packages for changed code") print(f"Testing {len(packages)} affected Go packages", flush=True) - subprocess.run(["go", "test", "-timeout=10m"] + (["-race"] if race else []) + packages, + subprocess.run(["go", "test", "-timeout=20m"] + (["-race"] if race else []) + packages, cwd=root, check=True) diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 280b28bf0..0794c20b3 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -145,6 +145,24 @@ def test_docs_do_not_start_go(self): planner.run_unit_tests(self.root, ["README.md"], CONFIG) run.assert_not_called() + def test_native_run_keeps_package_selection_race_and_full_package_budget(self): + inventory = subprocess.CompletedProcess([], 0, "\n".join(json.dumps(p) for p in self.packages)) + with patch.object(planner.subprocess, "run", return_value=inventory) as run: + planner.run_unit_tests(self.root, ["lib/lib_test.go"], CONFIG, race=True) + command = run.call_args.args[0] + self.assertEqual(command[:2], ["go", "test"]) + self.assertIn("-race", command) + self.assertEqual([arg for arg in command[2:] if not arg.startswith("-")], ["example/lib"]) + timeout = next(arg for arg in command if arg.startswith("-timeout=")) + package_minutes = int(timeout.removeprefix("-timeout=").removesuffix("m")) + # Runtime-backed tests must fit the same 20-minute package budget as + # the full CI lane, with extra wall time for setup and compilation. + self.assertGreaterEqual(package_minutes, 20) + fast = (ROOT / ".github/workflows/ci.yml").read_text() + unit = fast.split(" unit:\n", 1)[1].split("\n scripts:\n", 1)[0] + job_minutes = int(re.search(r"timeout-minutes: (\d+)", unit).group(1)) + self.assertGreaterEqual(job_minutes, package_minutes + 5) + class GitDiffTests(unittest.TestCase): def test_diff_includes_worktree_deletions_both_sides_of_renames_and_untracked_files(self): From 74a1d05f260ddf03c588e3da6d5cfb9b4a2027af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 10:30:09 +0100 Subject: [PATCH 05/14] Run conditional CI checks when their workflow changes --- scripts/ci_test_plan.py | 4 ++-- scripts/test_ci_test_plan.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py index a14ffcbb3..46c244823 100644 --- a/scripts/ci_test_plan.py +++ b/scripts/ci_test_plan.py @@ -72,8 +72,8 @@ def make_plan(files, config): "go": files is None or bool(relevant), "ddto": files is None or any(path.startswith("ddto/") or path in ( "main.go", "go.mod", "go.sum", "internal/cmd/instrument.go", - ".github/workflows/go-instrumentation.yml") for path in relevant), - "evaluator": files is None or any(path.startswith("skills/ddtest-onboarding-eval/") + ".github/workflows/ci.yml", ".github/workflows/go-instrumentation.yml") for path in relevant), + "evaluator": files is None or any(path == ".github/workflows/ci.yml" or path.startswith("skills/ddtest-onboarding-eval/") for path in files), "suites": sorted(selected), "integrations": [entry for entry in checks if entry["runtime"] != "playwright"], diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 0794c20b3..86031f669 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -94,6 +94,12 @@ def test_instrumentation_changes_enable_native_module_checks(self): for path in ("ddto/internal/runner/run.go", "main.go", "go.mod", "internal/cmd/instrument.go"): self.assertTrue(planner.make_plan([path], CONFIG)["ddto"]) + def test_fast_workflow_changes_execute_its_conditional_checks(self): + plan = planner.make_plan([".github/workflows/ci.yml"], CONFIG) + self.assertTrue(plan["go"]) + self.assertTrue(plan["ddto"]) + self.assertTrue(plan["evaluator"]) + def test_instrumentation_module_changes_do_not_run_other_language_frameworks(self): plan = planner.make_plan(["ddto/internal/runner/run.go"], CONFIG) self.assertTrue(plan["ddto"] and plan["go"]) From 26457383045a09b4f8ed814676d944b4ab4e1851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 10:45:54 +0100 Subject: [PATCH 06/14] Handle detached checkouts in the evaluator Git fixture --- .../scripts/test_review.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/skills/ddtest-onboarding-eval/scripts/test_review.py b/skills/ddtest-onboarding-eval/scripts/test_review.py index 71a64f9e4..cd580088f 100644 --- a/skills/ddtest-onboarding-eval/scripts/test_review.py +++ b/skills/ddtest-onboarding-eval/scripts/test_review.py @@ -190,22 +190,30 @@ def test_malformed_nested_report_still_finalizes_evidence_and_cleanup(self): self.assertIn('Invalid report fields', manifest['collection_errors'][0]) def test_ref_outside_branches_and_tags_is_fetched(self): - source = self.case.root / 'source.git' def git(*args): return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() - git('clone', '--bare', '--depth=1', '--single-branch', '--no-tags', '--', - Path(self.case.source).as_uri(), str(source)) - expected = git('-C', str(source), 'rev-parse', 'HEAD') - branch = git('-C', str(source), 'symbolic-ref', 'HEAD') - git('-C', str(source), 'update-ref', 'refs/pull/1/head', expected) - # CI has a depth-one checkout. Remove branch reachability without - # requiring a parent commit or creating an unsigned fixture commit. - git('-C', str(source), 'update-ref', '-d', branch) - self.case.args.repo = str(source) - self.case.args.ref = 'refs/pull/1/head' - code, manifest = self.case.trial() - self.assertEqual(code, 0, manifest) - self.assertEqual(manifest['commit'], expected) + for detached in (False, True): + with self.subTest(detached=detached): + source = self.case.root / f'source-{detached}.git' + git('clone', '--bare', '--depth=1', '--single-branch', '--no-tags', '--', + Path(self.case.source).as_uri(), str(source)) + expected = git('-C', str(source), 'rev-parse', 'HEAD') + branch = 'refs/heads/juan-fernandez/evaluator-fixture' + git('-C', str(source), 'update-ref', branch, expected) + git('-C', str(source), 'symbolic-ref', 'HEAD', branch) + if detached: + git('-C', str(source), 'update-ref', '--no-deref', 'HEAD', expected) + git('-C', str(source), 'update-ref', 'refs/pull/1/head', expected) + # A PR checkout can have detached HEAD. Remove every branch + # without requiring symbolic HEAD, a parent or a new commit. + for ref in git('-C', str(source), 'for-each-ref', '--format=%(refname)', 'refs/heads').splitlines(): + git('-C', str(source), 'update-ref', '-d', ref) + self.case.args.repo = str(source) + self.case.args.ref = 'refs/pull/1/head' + self.case.args.output = self.case.root / f'evidence-{detached}' + code, manifest = self.case.trial() + self.assertEqual(code, 0, manifest) + self.assertEqual(manifest['commit'], expected) def test_cleanup_requires_bound_marker_and_preserves_evidence_and_other_roots(self): code,manifest=self.case.trial() From 04f3ac0dcec6fb4eb0e288fca1b3d4b414ac6f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 11:14:03 +0100 Subject: [PATCH 07/14] Avoid Java smokes for JavaScript-only changes --- scripts/ci_test_plan.json | 6 +++--- scripts/test_ci_test_plan.py | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index 66a985ba4..eaa523a3a 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -119,12 +119,12 @@ ], "inputs": [ "internal/framework/jvm*", - "internal/platform/java*", + "internal/platform/java[._]*", "internal/gradle/**", "internal/maven/**", - "internal/onboard/java*", + "internal/onboard/java[._]*", "internal/onboard/instructions/java*", - "internal/testdrive/java*" + "internal/testdrive/java[._]*" ], "smoke": [ { diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 86031f669..64b7847de 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -59,14 +59,26 @@ def test_javascript_dependencies_propagate_to_every_javascript_consumer(self): self.assertEqual(set(plan["suites"]), {"javascript", "jest", "vitest", "mocha", "cypress", "playwright", "cucumber", "webdriverio", "onboarding"}) - def test_javascript_testdrive_sources_select_every_javascript_consumer(self): + def test_javascript_sources_select_only_javascript_consumers(self): consumers = {"javascript", "jest", "vitest", "mocha", "cypress", "playwright", "cucumber", "webdriverio", "onboarding"} - for path in ("internal/testdrive/javascript.go", "internal/testdrive/javascript_language.go"): + for directory in ("platform", "onboard", "testdrive"): + for name in ("javascript.go", "javascript_language.go"): + path = f"internal/{directory}/{name}" + with self.subTest(path=path): + self.assertEqual(set(planner.make_plan([path], CONFIG)["suites"]), consumers) + + def test_java_sources_still_select_both_build_tool_smokes(self): + for path, suites in ( + ("internal/platform/java.go", {"java"}), + ("internal/onboard/java_ci.go", {"java", "onboarding"}), + ("internal/testdrive/java_validation.go", {"java"}), + ): with self.subTest(path=path): plan = planner.make_plan([path], CONFIG) - self.assertTrue(consumers.issubset(plan["suites"])) - self.assertFalse({"python", "ruby"}.intersection(plan["suites"])) + self.assertEqual(set(plan["suites"]), suites) + self.assertTrue({"java-maven", "java-gradle"}.issubset( + entry["id"] for entry in plan["integrations"])) def test_cross_framework_onboarding_helpers_select_all_smokes(self): for path in ("internal/onboard/jest_command.go", "internal/onboard/python_handwritten.go"): From db76f6b46d15a140375f80f22133fd877f4a5328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 11:56:20 +0100 Subject: [PATCH 08/14] Track the shared testdrive execution-command helper --- scripts/ci_test_plan.json | 1 + scripts/test_ci_test_plan.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index eaa523a3a..5f9613e2b 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -10,6 +10,7 @@ ], "shared": [ "internal/testdrive/jest*.go", + "internal/testdrive/mocha.go", "internal/onboard/jest_command.go", "internal/onboard/python_handwritten.go" ], diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 64b7847de..7f4c04697 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -28,7 +28,8 @@ def test_python_change_selects_only_python_smoke(self): def test_shared_unknown_build_and_deleted_inputs_select_every_smoke(self): for path in ("internal/ext/command.go", "internal/onboard/ci_shell_state.go", - "internal/testdrive/jest_features.go", "internal/testdrive/new_fixture.json", + "internal/testdrive/jest_features.go", "internal/testdrive/mocha.go", + "internal/testdrive/new_fixture.json", "go.mod", "go.sum", ".github/workflows/ci.yml"): with self.subTest(path=path): plan = planner.make_plan([path], CONFIG) From 6d9b91c9224c4eb933bd4fc39ac67d37ab0a8c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 21:45:26 +0100 Subject: [PATCH 09/14] Select all smoke consumers of shared coverage helpers --- scripts/ci_test_plan.json | 5 +++-- scripts/test_ci_test_plan.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index 5f9613e2b..dd7fdb694 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -9,6 +9,8 @@ "ddto/README.md" ], "shared": [ + "internal/testdrive/cucumber_coverage.go", + "internal/testdrive/cypress_coverage.go", "internal/testdrive/jest*.go", "internal/testdrive/mocha.go", "internal/onboard/jest_command.go", @@ -316,8 +318,7 @@ "internal/onboard/cypress*", "internal/onboard/instructions/cypress.md", "internal/testdrive/cypress*", - "internal/testdrive/scripts/cypress*", - "internal/testdrive/cucumber_coverage.go" + "internal/testdrive/scripts/cypress*" ], "smoke": [ { diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 7f4c04697..f3b09b385 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -89,7 +89,8 @@ def test_cross_framework_onboarding_helpers_select_all_smokes(self): def test_cross_framework_coverage_and_path_helpers_select_their_consumers(self): for path, expected in ( - ("internal/testdrive/cucumber_coverage.go", {"cucumber", "cypress", "onboarding"}), + ("internal/testdrive/cucumber_coverage.go", set(CONFIG["suites"])), + ("internal/testdrive/cypress_coverage.go", set(CONFIG["suites"])), ("internal/framework/cypress.go", {"cypress", "playwright"}), ): with self.subTest(path=path): From 42cd8389cc73755b86514c0e003bbe27985f623d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 22:03:17 +0100 Subject: [PATCH 10/14] Preserve per-smoke integration timeout budgets --- .github/workflows/ci.yml | 4 ++-- docs/ci-test-selection.md | 2 ++ scripts/ci_smoke.py | 2 +- scripts/ci_test_plan.json | 13 +++++++++++++ scripts/test_ci_smoke.py | 7 +++++++ scripts/test_ci_test_plan.py | 19 +++++++++++++++++++ 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afadd6fa2..ada8e7a78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: needs: plan if: needs.plan.outputs.integrations != '[]' runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 strategy: fail-fast: false matrix: @@ -190,7 +190,7 @@ jobs: needs: plan if: needs.plan.outputs.playwright == 'true' runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 17 container: image: mcr.microsoft.com/playwright:v1.55.1-noble steps: diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index 13206017c..a713791a4 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -71,6 +71,8 @@ failure. Python, Ruby, Java and Vitest use their existing validation journeys; Java covers both Maven and Gradle. PRs use one pinned framework/tracer/runtime combination per journey. Older versions, `latest`, remaining scenarios, SDK parity and platform differences are checked by Full compatibility. +Each smoke retains its full compatibility lane's Go test timeout, with at least +five additional minutes in the job limit for setup and compilation. The `pr-test-plan` artifact and job summary show the selection. Every smoke requires explicit named Go test cases to report `pass`: a filtered, missing or diff --git a/scripts/ci_smoke.py b/scripts/ci_smoke.py index 90feba8cc..89e93c45c 100644 --- a/scripts/ci_smoke.py +++ b/scripts/ci_smoke.py @@ -45,7 +45,7 @@ def run_smoke(entry): for command in entry.get("setup", []): subprocess.run(command, check=True, env=env) command = (["xvfb-run", "-a"] if entry.get("xvfb") else []) + [ - "go", "test", "-json", "-count=1", "-timeout=8m", "-run", entry["tests"], + "go", "test", "-json", "-count=1", f"-timeout={entry['timeout_minutes']}m", "-run", entry["tests"], ] + entry.get("packages", ["./internal/testdrive"]) run_tests(command, env, entry["required"]) diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index dd7fdb694..07d068042 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -37,6 +37,7 @@ { "id": "python", "runtime": "python", + "timeout_minutes": 20, "tests": "^TestPublicPythonRubyTestdrives$/^pytest$", "required": [ "TestPublicPythonRubyTestdrives/pytest" @@ -71,6 +72,7 @@ { "id": "ruby", "runtime": "ruby", + "timeout_minutes": 20, "tests": "^TestRubyValidationIntegration$/^(rspec|minitest)$", "required": [ "TestRubyValidationIntegration/rspec", @@ -133,6 +135,7 @@ { "id": "java-maven", "runtime": "java", + "timeout_minutes": 20, "tests": "^TestJavaValidationIntegration$/^junit4$", "required": [ "TestJavaValidationIntegration/junit4" @@ -146,6 +149,7 @@ { "id": "java-gradle", "runtime": "gradle", + "timeout_minutes": 20, "tests": "^TestJavaValidationIntegration$/^gradle_junit4$", "required": [ "TestJavaValidationIntegration/gradle_junit4" @@ -175,6 +179,7 @@ { "id": "javascript", "runtime": "node", + "timeout_minutes": 10, "packages": [ "./internal/compatibility" ], @@ -210,6 +215,7 @@ { "id": "jest", "runtime": "node", + "timeout_minutes": 12, "tests": "^TestPublicJestTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicJestTestdrive/passing", @@ -247,6 +253,7 @@ { "id": "vitest", "runtime": "node", + "timeout_minutes": 15, "tests": "^TestPublicVitestRunnerTestdrive$/^mixed_CI_paths$", "required": [ "TestPublicVitestRunnerTestdrive/mixed_CI_paths" @@ -286,6 +293,7 @@ { "id": "mocha", "runtime": "node", + "timeout_minutes": 10, "tests": "^TestPublicMochaTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicMochaTestdrive/passing", @@ -324,6 +332,7 @@ { "id": "cypress", "runtime": "node", + "timeout_minutes": 12, "tests": "^TestPublicCypressTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicCypressTestdrive/passing", @@ -364,6 +373,7 @@ { "id": "playwright", "runtime": "playwright", + "timeout_minutes": 12, "tests": "^TestPublicPlaywrightTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicPlaywrightTestdrive/passing", @@ -400,6 +410,7 @@ { "id": "cucumber", "runtime": "node", + "timeout_minutes": 12, "tests": "^TestPublicCucumberTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicCucumberTestdrive/passing", @@ -439,6 +450,7 @@ { "id": "webdriverio", "runtime": "node", + "timeout_minutes": 12, "tests": "^TestPublicWebdriverIOTestdrive$/^(passing|existing_failure)$", "required": [ "TestPublicWebdriverIOTestdrive/passing", @@ -483,6 +495,7 @@ { "id": "onboarding", "runtime": "node", + "timeout_minutes": 8, "tests": "^TestPublicOnboardApply$", "required": [ "TestPublicOnboardApply" diff --git a/scripts/test_ci_smoke.py b/scripts/test_ci_smoke.py index 0e56ef350..c87e97b61 100644 --- a/scripts/test_ci_smoke.py +++ b/scripts/test_ci_smoke.py @@ -9,6 +9,13 @@ class SmokeTests(unittest.TestCase): + def test_smoke_passes_its_runtime_budget_to_go(self): + for minutes in (8, 12, 20): + with self.subTest(minutes=minutes), patch.object(ci_smoke, "run_tests") as run: + ci_smoke.run_smoke({"tests": "^TestSmoke$", "required": ["TestSmoke"], + "timeout_minutes": minutes}) + self.assertIn(f"-timeout={minutes}m", run.call_args.args[0]) + def test_missing_skipped_or_filtered_cases_cannot_pass(self): for passed in (set(), {"TestSmoke"}, {"TestSmoke/other"}): with self.assertRaises(ValueError): diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index f3b09b385..1293b07df 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -230,6 +230,25 @@ def test_fast_smokes_have_unique_ids_and_explicit_required_cases(self): self.assertTrue(entry["required"]) self.assertTrue(entry["tests"].startswith("^Test")) + def test_smokes_preserve_integration_budgets_and_allow_job_setup_time(self): + # These are the existing full lanes' Go test allowances, including Go's + # default ten minutes for the JavaScript platform compatibility tests. + budgets = {"python": 20, "ruby": 20, "java-maven": 20, "java-gradle": 20, + "javascript": 10, "jest": 12, "vitest": 15, "mocha": 10, + "cypress": 12, "playwright": 12, "cucumber": 12, + "webdriverio": 12, "onboarding": 8} + workflow = (ROOT / ".github/workflows/ci.yml").read_text() + integration = workflow.split(" integration:\n", 1)[1].split("\n playwright:\n", 1)[0] + playwright = workflow.split(" playwright:\n", 1)[1].split("\n evaluator-isolation:\n", 1)[0] + for suite in CONFIG["suites"].values(): + for entry in suite["smoke"]: + with self.subTest(smoke=entry["id"]): + minutes = entry["timeout_minutes"] + self.assertGreaterEqual(minutes, budgets[entry["id"]]) + job = playwright if entry["runtime"] == "playwright" else integration + job_minutes = int(re.search(r"timeout-minutes: (\d+)", job).group(1)) + self.assertGreaterEqual(job_minutes, minutes + 5) + def test_fast_units_provision_and_require_project_runtimes(self): fast = (ROOT / ".github/workflows/ci.yml").read_text() unit = fast.split(" unit:\n", 1)[1].split("\n scripts:\n", 1)[0] From fc91e9cb9c87eb6da6fe1a0957563267452a8c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 4 Nov 2026 23:05:40 +0100 Subject: [PATCH 11/14] Select framework smokes for runtime-backed test edits --- docs/ci-test-selection.md | 6 ++++-- scripts/ci_test_plan.json | 14 ++++++++++++++ scripts/ci_test_plan.py | 3 ++- scripts/test_ci_test_plan.py | 32 ++++++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index a713791a4..fb0e24fe8 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -57,8 +57,10 @@ embedded instructions/scripts, JavaScript dependencies and smoke recipes. Python-only changes select Python; JavaScript platform changes also select all JavaScript frameworks. Shared or unclassified production inputs select every PR smoke. Instrumentation module changes select its units and the root/release -smoke without unrelated framework integrations. Unit-test-only edits omit -external smokes; integration-test edits still select their framework. +smoke without unrelated framework integrations. Pure unit-test edits omit +external smokes. Integration tests and ordinary `_test.go` files registered in +`runtime_test_inputs` select their framework smokes. Register runtime-backed test +files there when adding tests that need framework fixtures beyond the unit job. The native Go job installs Python/pytest/ddtrace, Ruby/Bundler and Node/Yarn, and sets `DDTEST_REQUIRE_PROJECT_RUNTIMES=1` so focused runtime regressions in selected packages fail on missing prerequisites instead of silently skipping. diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json index 07d068042..b05b5461d 100644 --- a/scripts/ci_test_plan.json +++ b/scripts/ci_test_plan.json @@ -16,6 +16,20 @@ "internal/onboard/jest_command.go", "internal/onboard/python_handwritten.go" ], + "runtime_test_inputs": [ + "**/*integration_test.go", + "internal/compatibility/**", + "internal/framework/webdriverio_test.go", + "internal/platform/javascript_test.go", + "internal/testdrive/cucumber_support_test.go", + "internal/testdrive/jest_preflight_test.go", + "internal/testdrive/mocha_validation_test.go", + "internal/testdrive/vitest_review_test.go", + "internal/testdrive/vitest_test.go", + "internal/testdrive/webdriverio_followup_test.go", + "internal/testdrive/webdriverio_review_test.go", + "internal/testdrive/webdriverio_test.go" + ], "suites": { "python": { "jobs": [ diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py index 46c244823..9f211808c 100644 --- a/scripts/ci_test_plan.py +++ b/scripts/ci_test_plan.py @@ -61,7 +61,8 @@ def make_plan(files, config): # binary smoke cover changes there; they do not affect framework runners. if path.startswith("ddto/"): continue - if path.endswith("_test.go") and not path.endswith("integration_test.go") and not path.startswith("internal/compatibility/"): + if path.endswith("_test.go") and not any(fnmatchcase(path, pattern) + for pattern in config["runtime_test_inputs"]): continue selected.update(affected_suites(path, config)) checks = [{"id": entry["id"], "runtime": entry["runtime"]} diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py index 1293b07df..45e1db69d 100644 --- a/scripts/test_ci_test_plan.py +++ b/scripts/test_ci_test_plan.py @@ -96,14 +96,42 @@ def test_cross_framework_coverage_and_path_helpers_select_their_consumers(self): with self.subTest(path=path): self.assertEqual(set(planner.make_plan([path], CONFIG)["suites"]), expected) - def test_mocha_fixture_helper_tracks_webdriverio_without_expanding_unit_only_checks(self): + def test_mocha_runtime_fixture_helper_tracks_mocha_and_webdriverio_smokes(self): path = "internal/testdrive/mocha_validation_test.go" self.assertEqual(planner.affected_suites(path, CONFIG), {"mocha", "webdriverio", "onboarding"}) plan = planner.make_plan([path], CONFIG) self.assertTrue(plan["go"]) - self.assertEqual(plan["integrations"], []) + self.assertEqual({entry["id"] for entry in plan["integrations"]}, + {"mocha", "webdriverio", "onboarding"}) self.assertFalse(plan["playwright"]) + def test_runtime_backed_test_edits_select_their_framework_smokes(self): + for path, expected in ( + ("internal/testdrive/vitest_review_test.go", {"vitest", "onboarding"}), + ("internal/testdrive/vitest_test.go", {"vitest", "onboarding"}), + ("internal/testdrive/jest_preflight_test.go", {"jest"}), + ("internal/testdrive/cucumber_support_test.go", {"cucumber", "onboarding"}), + ("internal/framework/webdriverio_test.go", {"webdriverio", "onboarding"}), + ("internal/testdrive/webdriverio_followup_test.go", {"webdriverio", "onboarding"}), + ("internal/testdrive/webdriverio_review_test.go", {"webdriverio", "onboarding"}), + ("internal/testdrive/webdriverio_test.go", {"webdriverio", "onboarding"}), + ("internal/platform/javascript_test.go", {"javascript", "jest", "vitest", "mocha", + "cypress", "playwright", "cucumber", "webdriverio", "onboarding"}), + ): + with self.subTest(path=path): + plan = planner.make_plan([path], CONFIG) + self.assertEqual(set(plan["suites"]), expected) + self.assertTrue(plan["go"]) + + def test_runtime_fixture_gates_cannot_be_classified_as_unit_only(self): + gate = re.compile(r'os\.Getenv\("DDTEST_(?:[A-Z_]+_NODE_MODULES|RUN_[A-Z_]+_INTEGRATION_TEST)"\)') + for path in (ROOT / "internal").rglob("*_test.go"): + if gate.search(path.read_text()): + relative = path.relative_to(ROOT).as_posix() + with self.subTest(path=relative): + plan = planner.make_plan([relative], CONFIG) + self.assertTrue(plan["integrations"] or plan["playwright"]) + def test_instrumentation_changes_enable_native_module_checks(self): for path in ("ddto/internal/runner/run.go", "main.go", "go.mod", "internal/cmd/instrument.go"): self.assertTrue(planner.make_plan([path], CONFIG)["ddto"]) From a5640615ded30238d916c8f4b7fc14b00cf15665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 14:43:46 +0100 Subject: [PATCH 12/14] Reuse CI fixtures while preserving the existing test suite --- .github/actions/node-test-fixture/action.yml | 48 + .github/workflows/ci-full.yml | 1132 -------------- .github/workflows/ci.yml | 1315 ++++++++++++++--- .github/workflows/go-instrumentation.yml | 20 +- .github/workflows/release.yml | 19 +- Makefile | 4 +- ddto/AGENTS.md | 6 +- ddto/docs/validation.md | 33 +- docs/ci-performance.md | 49 + docs/ci-test-selection.md | 106 -- internal/releaseworkflow/ci_shards_test.go | 37 +- internal/releaseworkflow/node_fixture_test.go | 70 + .../apply_cucumber_integration_test.go | 6 +- .../testdrive/apply_mocha_integration_test.go | 6 +- .../apply_webdriverio_integration_test.go | 6 +- .../apply_workflow_integration_test.go | 6 +- .../configurations_integration_test.go | 6 +- .../coordinator_workflow_integration_test.go | 8 +- .../cucumber_workflow_integration_test.go | 6 +- .../cypress_workflow_integration_test.go | 6 +- ...orwarded_configuration_integration_test.go | 6 +- .../testdrive/frameworks_integration_test.go | 4 +- .../java_validation_integration_test.go | 4 +- .../jest_workflow_integration_test.go | 8 +- .../testdrive/languages_integration_test.go | 4 +- .../loader_configuration_integration_test.go | 6 +- .../mocha_workflow_integration_test.go | 6 +- .../package_configuration_integration_test.go | 9 +- .../playwright_workflow_integration_test.go | 6 +- internal/testdrive/preflight_review_test.go | 2 +- internal/testdrive/python_feedback_test.go | 26 +- .../testdrive/ruby_fork_integration_test.go | 4 +- .../testdrive/ruby_nested_integration_test.go | 4 +- .../ruby_receipt_integration_test.go | 4 +- .../ruby_validation_integration_test.go | 4 +- internal/testdrive/vitest_review_test.go | 2 +- .../vitest_runner_integration_test.go | 4 +- .../webdriverio_workflow_integration_test.go | 6 +- internal/testutil/binary.go | 65 + internal/testutil/binary_test.go | 48 + scripts/ci_smoke.py | 65 - scripts/ci_test_plan.json | 528 ------- scripts/ci_test_plan.py | 171 --- scripts/test_ci_cucumber_matrix.py | 6 +- scripts/test_ci_smoke.py | 42 - scripts/test_ci_test_plan.py | 316 ---- 46 files changed, 1596 insertions(+), 2643 deletions(-) create mode 100644 .github/actions/node-test-fixture/action.yml delete mode 100644 .github/workflows/ci-full.yml create mode 100644 docs/ci-performance.md delete mode 100644 docs/ci-test-selection.md create mode 100644 internal/releaseworkflow/node_fixture_test.go create mode 100644 internal/testutil/binary.go create mode 100644 internal/testutil/binary_test.go delete mode 100644 scripts/ci_smoke.py delete mode 100644 scripts/ci_test_plan.json delete mode 100644 scripts/ci_test_plan.py delete mode 100644 scripts/test_ci_smoke.py delete mode 100644 scripts/test_ci_test_plan.py diff --git a/.github/actions/node-test-fixture/action.yml b/.github/actions/node-test-fixture/action.yml new file mode 100644 index 000000000..8904fd374 --- /dev/null +++ b/.github/actions/node-test-fixture/action.yml @@ -0,0 +1,48 @@ +name: Locked Node test fixture +description: Reuse installed test dependencies only for an identical lockfile and runtime. +inputs: + directory: + description: Absolute fixture directory. + required: true + dependencies: + description: Space-separated npm dependencies, or empty for a supplied lockfile. + default: '' +runs: + using: composite + steps: + - name: Resolve the fixture and identify its complete dependency graph + id: lock + shell: bash + env: + FIXTURE_DIRECTORY: ${{ inputs.directory }} + FIXTURE_DEPENDENCIES: ${{ inputs.dependencies }} + run: | + set -euo pipefail + mkdir -p "$FIXTURE_DIRECTORY" + cd "$FIXTURE_DIRECTORY" + if [ -n "$FIXTURE_DEPENDENCIES" ]; then + printf '%s\n' '{"name":"ddtest-ci-fixture","version":"1.0.0","private":true}' > package.json + read -ra dependencies <<< "$FIXTURE_DEPENDENCIES" + npm install --package-lock-only --ignore-scripts --save-exact --no-audit --no-fund "${dependencies[@]}" + fi + lock=$(sha256sum "$FIXTURE_DIRECTORY/package-lock.json" | cut -d ' ' -f 1) + echo "key=$(node --version)-$(npm --version)-$lock" >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + id: modules + with: + path: ${{ inputs.directory }}/node_modules + key: ${{ runner.os }}-${{ runner.arch }}-node-test-fixture-${{ steps.lock.outputs.key }} + - name: Install a missing fixture from its lockfile + if: steps.modules.outputs.cache-hit != 'true' + shell: bash + env: + FIXTURE_DIRECTORY: ${{ inputs.directory }} + run: | + cd "$FIXTURE_DIRECTORY" + npm ci --no-audit --no-fund + # Save the prepared dependencies before tests can write framework caches. + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + if: steps.modules.outputs.cache-hit != 'true' + with: + path: ${{ inputs.directory }}/node_modules + key: ${{ runner.os }}-${{ runner.arch }}-node-test-fixture-${{ steps.lock.outputs.key }} diff --git a/.github/workflows/ci-full.yml b/.github/workflows/ci-full.yml deleted file mode 100644 index 07fbe62a6..000000000 --- a/.github/workflows/ci-full.yml +++ /dev/null @@ -1,1132 +0,0 @@ -name: Full compatibility - -on: - workflow_call: - workflow_dispatch: - -permissions: - contents: read - actions: read - -jobs: - fast-validation: - uses: ./.github/workflows/ci.yml - secrets: inherit - - go-instrumentation: - uses: ./.github/workflows/go-instrumentation.yml - with: - compatibility: full - secrets: inherit - - test-support: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 - - - name: Test onboarding evaluator - run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' - - # install.sh, and with the runner's pwsh install.ps1, against local release files. - - name: Test install scripts - run: python -m unittest discover -s scripts -p 'test_*.py' - - test-shards: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - # This job already has a compiler-specific cache below. Restoring - # setup-go's cache too collides with read-only module files. - cache: false - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Cache Go modules - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 - - # ddto is its own module, checked by its own workflow; its benchmark - # fixtures are deliberately compact. - - name: Format check - run: | - unformatted="$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" - if [ -n "$unformatted" ]; then - echo "The following files are not formatted correctly:" - echo "$unformatted" - exit 1 - fi - - - name: Vet - run: go vet ./... - - - name: Select Go test shard - run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" - - - name: Configure Datadog Test Optimization - if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} - uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 - with: - languages: go - # Pin upstream Go 1.27 compatibility fixes until they reach a stable - # Orchestrion release. - go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 - api_key: ${{ secrets.DATADOG_API_KEY }} - site: datadoghq.com - service: ddtest - - - name: Run tests - run: go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... - env: - DD_ENV: ci - DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" - - - name: Retain shard coverage - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: unit-coverage-${{ matrix.shard }} - path: coverage.out - if-no-files-found: error - overwrite: true - retention-days: 1 - - # Keep the existing required check name; a skipped, cancelled or failed - # shard/support job must not turn the aggregate green. - test: - if: ${{ always() }} - needs: [test-shards, test-support] - runs-on: ubuntu-latest - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - steps: - - name: Require every test job - env: - SHARDS_RESULT: ${{ needs.test-shards.result }} - SUPPORT_RESULT: ${{ needs.test-support.result }} - run: | - test "$SHARDS_RESULT" = success - test "$SUPPORT_RESULT" = success - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: unit-coverage-* - path: ${{ runner.temp }}/unit-coverage - - name: Merge complete coverage - run: python3 scripts/go_test_shards.py merge --expected 3 --output coverage.out "$RUNNER_TEMP"/unit-coverage/*/coverage.out - - name: Upload coverage report to Datadog - if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} - uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 # v1.0.5 - with: - api_key: ${{ secrets.DATADOG_API_KEY }} - site: datadoghq.com - files: coverage.out - format: go-coverprofile - version: v5.19.0 - - # Dogfooding: runs the `test` suite with ddtest instrument, built from this - # checkout as a release builds it, instead of Orchestrion. Its results reach - # Test Optimization beside those of `test`, under this job's name. - test-ddtest-instrument-shards: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - env: - DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - # This job already has a compiler-specific cache below. Restoring - # setup-go's cache too collides with read-only module files. - cache: false - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Set up Node.js for project environment regression tests - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Install Yarn for the Plug'n'Play regression fixture - run: npm install --global yarn@1.22.22 - - - name: Set up Ruby for project environment regression tests - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - # Its own key, so that its instrumented builds don't replace those of `test`. - - name: Cache Go modules - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- - - - name: Download dependencies - run: go mod download - - - name: Verify dependencies - run: go mod verify - - - name: Install Python test dependencies - run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - - name: Install evaluator package-manager fixture - run: npm install --global pnpm@12.14.0 - - - name: Select Go test shard - run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" - - # The release builder, as `make release` runs it for one target: ddtest - # with the runtime sources and tool wrapper that ddtest instrument provides. - - name: Build ddtest instrument from this checkout - run: | - version="dogfood-${GITHUB_SHA::12}" - go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ - -version "v0.0.0-$version" -ldflags "-X github.com/DataDog/ddtest/internal/buildinfo.Version=$version" \ - -target linux/amd64 -o "$RUNNER_TEMP/dist" - mkdir -p "$RUNNER_TEMP/bin" - mv "$RUNNER_TEMP/dist/ddtest-linux-amd64" "$RUNNER_TEMP/bin/ddtest" - echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" - "$RUNNER_TEMP/bin/ddtest" instrument --version - - - name: Run tests - run: ddtest instrument go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... - env: - DD_CIVISIBILITY_ENABLED: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' && 'parent' || 'false' }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_SERVICE: ddtest - DD_ENV: ci - JOB_CHECK_RUN_ID: ${{ job.check_run_id }} - DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" - # `test` uploads this commit's coverage report. - DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED: "false" - - test-ddtest-instrument: - if: ${{ always() }} - needs: test-ddtest-instrument-shards - runs-on: ubuntu-latest - steps: - - name: Require every instrumented shard - env: - SHARDS_RESULT: ${{ needs.test-ddtest-instrument-shards.result }} - run: test "$SHARDS_RESULT" = success - - rspec-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - rspec: "3.12.0" - datadog_ci: "1.31.0" - - rspec: "3.13.2" - datadog_ci: "1.37.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Ruby - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Test RSpec adapter - run: | - gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document - gem install rspec --version "${{ matrix.rspec }}" --no-document - DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ - DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$' - DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$' - - minitest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - minitest: "5.26.0" - datadog_ci: "1.31.0" - - minitest: "6.0.6" - datadog_ci: "1.37.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Ruby - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: "3.4" - - - name: Test Minitest adapter - run: | - gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document - gem install minitest --version "${{ matrix.minitest }}" --no-document - gem install rake --version "13.2.1" --no-document - DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ - DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ - go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$' - - pytest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - pytest: "8.4.2" - ddtrace: "4.11.0" - - pytest: "9.1.1" - ddtrace: "4.14.0" - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - - name: Test pytest adapter - run: | - python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" pytest-cov pytest-xdist - DDTEST_PYTHON_BINARY="$(command -v python)" \ - go test -v ./internal/compatibility -run '^TestPyTest(AdapterIntegration|SuiteModeSelectorBatches)$' - DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ - go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$' - DDTEST_PYTEST_PYTHON="$(command -v python)" \ - go test -v ./internal/testdrive -run '^TestPytest(RealValidation|CoverageExecutesOnlyInsideCopy|LoggingExecutesOnlyInsideCopy|RejectsEditableImportsFromOriginalCheckout|CoverageDataExecutesOnlyInsideCopy|IgnoredConfigurationUsesRoot|ObserverKeepsConcurrentDuplicateExecutions|PreflightReportsSelectedEnvironment)$' - - javascript-platform-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - dd_trace: ["5.111.0", "6.15.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test JavaScript platform - run: | - dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" - npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" - DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestJavaScript(Platform|ActionPreload)Integration$' - - jest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - jest: ["27.5.1", "28.1.3", "29.4.3", "29.7.0", "30.5.1"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Jest adapter - run: | - jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" - npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" dd-trace@5.86.0 - DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestJest(Adapter|Testdrive)Integration$' - DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestJestCheckOnlyIntegration$' - - vitest-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.5"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Vitest adapter - run: | - vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" - # npm 10.9.8 crashes while resolving Vitest's peer dependencies. - # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. - npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "@vitest/coverage-v8@${{ matrix.vitest }}" "dd-trace@5.132.0" - DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' - DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestVitest(Configuration|ValidationArguments|MultiProjectRejectionKeepsConfigurationEvidence|RepeatedRunOptionIsNormalized)Integration$' - - mocha-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Mocha adapter - run: | - mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" - npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" - DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestMocha.*Integration$' - DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestMochaNativeObserverIntegration$' - - cypress-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Cypress adapter - run: | - cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}" - npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}" - DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ - DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ - DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ - go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$' - - playwright-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Playwright adapter - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" - run: | - playwright_dir="${RUNNER_TEMP}/playwright-${{ matrix.playwright }}" - npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}" - DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \ - DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$' - - cucumber-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: Set up Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "22" - - - name: Test Cucumber adapter - run: | - cucumber_dir="${RUNNER_TEMP}/cucumber-${{ matrix.cucumber }}" - npm install --prefix "${cucumber_dir}" "@cucumber/cucumber@${{ matrix.cucumber }}" - DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ - DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ - DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ - go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$' - DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ - go test -v ./internal/testdrive -run '^TestCucumberSupportMetadataIntegration$' - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - - name: golangci-lint - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 - with: - version-file: .golangci-lint-version - args: --timeout=5m - - jest-testdrive-workflow: - name: jest-testdrive-workflow (${{ matrix.node }}, ${{ matrix.jest }}, ${{ matrix.group }} ${{ matrix.shard }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # Per runtime: TestPublicJestTestdrive over two shards, plus two groups of - # configuration tests of similar length; together they are the full set. - include: - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} - - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} - # Unpinned: catches framework releases our pinned versions predate. - - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} - - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: ${{ matrix.node }} - - name: Install real Jest and tracer - run: | - npm install --prefix "${RUNNER_TEMP}/jest-testdrive" jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 - npm ls --prefix "${RUNNER_TEMP}/jest-testdrive" --depth=0 - - name: Validate paired outcomes, all feature probes and cleanup - env: - DDTEST_JEST_NODE_MODULES: ${{ runner.temp }}/jest-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(${{ matrix.tests }})$' -count=1 -timeout=12m - - windows-probe-diagnostics: - runs-on: windows-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - name: Test Windows probe filesystem diagnostics - run: go test ./internal/testdrive -run '^(TestProbePublicationExplainsWindowsFilesystemErrors|TestReportsCanReplaceExistingDestinationTwice)$' -count=1 - - - testdrive-language-workflow: - runs-on: ubuntu-latest - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - framework: [pytest] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: '3.12' - - name: Run Python testdrive - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - run: go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives/${{ matrix.framework }}$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: testdrive-${{ matrix.framework }} - path: ${{ runner.temp }}/ddtest-workflow-reports - - ruby-testdrive: - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - include: - - name: pinned-minitest-6 - datadog_ci: "1.40.0" - rspec: "3.13.2" - minitest: "6.0.6" - rake: "13.4.2" - - name: pinned-minitest-5 - datadog_ci: "1.40.0" - rspec: "3.13.2" - minitest: "5.27.0" - rake: "13.4.2" - - name: latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - with: - ruby-version: '3.4' - - name: Install Ruby validation dependencies - env: - DATADOG_CI: ${{ matrix.datadog_ci }} - RSPEC: ${{ matrix.rspec }} - MINITEST: ${{ matrix.minitest }} - RAKE: ${{ matrix.rake }} - run: | - install() { if [ -n "$2" ]; then gem install "$1" --version "$2" --no-document; else gem install "$1" --no-document; fi; } - install datadog-ci "$DATADOG_CI" - install rspec "$RSPEC" - install minitest "$MINITEST" - install rake "$RAKE" - gem install activesupport --conservative --no-document - gem list | grep -E '^(datadog-ci|rspec-core|minitest|rake|activesupport) ' - - name: Run Ruby testdrive integration tests - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_RUN_RUBY_INTEGRATION_TEST: '1' - DDTEST_RUN_RUBY_FORK_INTEGRATION_TEST: '1' - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - run: | - go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives$/^(rspec|minitest)$' -count=1 -timeout=20m - go test -v ./internal/testdrive -run '^TestRuby.+Integration$' -skip '^TestRubyValidationIntegration$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ruby-testdrive-${{ matrix.name }} - path: ${{ runner.temp }}/ddtest-workflow-reports - if-no-files-found: ignore - retention-days: 7 - - java-testdrive: - name: java-testdrive (${{ matrix.name }}, ${{ matrix.java }}, ${{ matrix.tracer }}, ${{ matrix.shard }}) - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - # Each scenario runs a full paired validation with real Maven or Gradle builds; - # the scenarios are spread over shards (DDTEST_TEST_SHARD). - name: [pinned, latest] - shard: ['1/4', '2/4', '3/4', '4/4'] - include: - - name: pinned - java: '17' - tracer: '1.67.1' - - name: latest - java: '21' - tracer: latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 - with: - distribution: temurin - java-version: ${{ matrix.java }} - # The Maven fixtures are generated by the test, so the repository has no - # pom.xml for setup-java's cache key; key the cache on the test instead. - - name: Cache Maven dependencies - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ matrix.java }}-${{ hashFiles('internal/testdrive/java_validation_integration_test.go') }} - restore-keys: | - ${{ runner.os }}-maven-${{ matrix.java }}- - # The Gradle fixtures run the gradle on PATH, as a CI job with this - # action does; it also caches the Gradle user home. - - uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 - with: - gradle-version: '8.10.2' - - name: Run Java testdrive integration tests - env: - DDTEST_RUN_JAVA_INTEGRATION_TEST: '1' - DDTEST_JAVA_TRACER_VERSION: ${{ matrix.tracer }} - DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports - DDTEST_TEST_SHARD: ${{ matrix.shard }} - # Every validation starts Maven many times; short-lived Maven JVMs - # start faster without the optimizing compiler. Both runs inherit it. - MAVEN_OPTS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 - run: go test -v ./internal/testdrive -run '^TestJavaValidationIntegration$' -count=1 -timeout=20m - - name: Retain failed testdrive reports - if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: java-testdrive-${{ matrix.name }}-${{ strategy.job-index }} - path: ${{ runner.temp }}/ddtest-workflow-reports - if-no-files-found: ignore - retention-days: 7 - - testdrive-vitest-workflow: - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - include: - - vitest: '3.2.7' - tracer: '5.132.0' - - vitest: '4.1.11' - tracer: '6.21.0' - - vitest: '5.0.5' - tracer: '6.21.0' - - vitest: 'latest' - tracer: '6.21.0' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Resolve Vitest version - run: | - version=$(npm view "vitest@${{ matrix.vitest }}" version) - echo "Vitest $version" - echo "DDTEST_VITEST_VERSION=$version" >> "$GITHUB_ENV" - - name: Validate Vitest CLI and Node API paths against local intake - env: - DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' - DDTEST_VITEST_TRACER_VERSION: ${{ matrix.tracer }} - run: go test -v ./internal/testdrive -run '^TestPublicVitestRunnerTestdrive$' -count=1 -timeout=15m - - onboard-apply-workflow: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Validate generated workflow edits with actionlint - run: | - go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 - DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/onboard -run '^TestGeneratedSetupWorkflow.*Actionlint$' -count=1 - - name: Install apply fixture dependencies - run: npm install --prefix "$RUNNER_TEMP/apply-testdrive" vitest@4.1.11 mocha@11.7.6 nyc@17.1.0 dd-trace@6.20.1 @cucumber/cucumber@12.2.0 @wdio/cli@9.20.0 @wdio/local-runner@9.20.0 @wdio/mocha-framework@9.20.0 --no-audit --no-fund - - name: Apply setup and validate against local intake - env: - DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_VITEST_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules - DDTEST_CHROME_BINARY: /usr/bin/google-chrome - run: DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/testdrive -run '^TestPublic(OnboardApply|MochaApply|WebdriverIOApply|CucumberApply)$' -count=1 -timeout=8m - - testdrive-mocha-workflow: - runs-on: ubuntu-latest - timeout-minutes: 12 - strategy: - fail-fast: false - matrix: - mocha: ['11.7.6', 'latest'] - shard: ['1/2', '2/2'] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Install Mocha testdrive fixture dependencies - run: | - npm install --prefix "$RUNNER_TEMP/mocha-testdrive" "mocha@${{ matrix.mocha }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.21.0 --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/mocha-testdrive" --depth=0 - - name: Validate CLI, programmatic runner, existing failures and coverage gate - env: - DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/mocha-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(TestPublicMochaTestdrive|TestMochaProbeCoverageThresholdControls|TestMochaDelayedCommandUsesSharedTimeout|TestMochaPreflightResolvesSelectedPackage)$' -count=1 -timeout=10m - - playwright-latest: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.resolve.outputs.version }} - steps: - - id: resolve - run: echo "version=$(npm view @playwright/test version)" >> "$GITHUB_OUTPUT" - - testdrive-playwright-workflow: - needs: playwright-latest - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - playwright: ['1.55.1', 'latest'] - shard: ['1/2', '2/2'] - # The official image ships the browsers and their system packages for this - # exact Playwright release, so nothing is downloaded or installed per run. - container: - image: mcr.microsoft.com/playwright:v${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }}-noble - env: - PLAYWRIGHT_VERSION: ${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - # The workspace is owned by the runner user, not the container's root; - # go build reads it with git for VCS stamping. - - name: Trust the workspace for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Isolate Playwright fixture caches - run: echo "npm_config_cache=$RUNNER_TEMP/playwright-testdrive/cache" >> "$GITHUB_ENV" - - name: Install Playwright testdrive fixture dependencies - run: | - npm install --prefix "$RUNNER_TEMP/playwright-testdrive" "@playwright/test@$PLAYWRIGHT_VERSION" dd-trace@6.20.1 --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/playwright-testdrive" --depth=0 - - name: Validate browser execution, existing failures and supervised cleanup - env: - DDTEST_PLAYWRIGHT_NODE_MODULES: ${{ runner.temp }}/playwright-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: go test -v ./internal/testdrive -run '^(TestPublicPlaywrightTestdrive|TestPlaywrightDiscoveryOwnsImportSideEffects|TestPlaywrightDiscoveryKeepsConfiguredReportsOutOfTheCheckout)$' -count=1 -timeout=12m - - testdrive-cypress-workflow: - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # Scenarios are split by integration_shard_test.go; each runs exactly - # once per Cypress version. - cypress: ['15.1.0', 'latest'] - shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Resolve Cypress version - id: cypress - run: echo "version=$(npm view "cypress@${{ matrix.cypress }}" version | tail -n 1)" >> "$GITHUB_OUTPUT" - # The Cypress binary downloads during npm install; reuse it per release. - - name: Cache Cypress binary - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - with: - path: ${{ runner.temp }}/cypress-testdrive-cache - key: ${{ runner.os }}-cypress-binary-${{ steps.cypress.outputs.version }} - - name: Install Cypress testdrive fixture dependencies - env: - CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache - run: | - npm install --prefix "$RUNNER_TEMP/cypress-testdrive" "cypress@${{ matrix.cypress }}" dd-trace@6.20.1 typescript@5.9.3 --save-exact --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/cypress-testdrive" --depth=0 - - name: Validate original CLI, action API and matching failures with TIA deferred - env: - DDTEST_CYPRESS_NODE_MODULES: ${{ runner.temp }}/cypress-testdrive/node_modules - CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: xvfb-run -a go test -v ./internal/testdrive -run '^(TestPublicCypressTestdrive|TestCypressTypeScriptWrapperTranspilesToCommonJS)$' -failfast -count=1 -timeout=12m - - # Resolve floating dependencies before expanding the matrix. Identical aliases - # share locked fixtures and jobs within this comprehensive run. - cucumber-matrix: - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Resolve versions, lock fixtures and select distinct configurations - id: plan - run: | - python3 scripts/ci_cucumber_matrix.py --output "$RUNNER_TEMP/cucumber-fixtures" \ - --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ci-cucumber-fixtures - path: ${{ runner.temp }}/cucumber-fixtures - if-no-files-found: error - retention-days: 14 - - testdrive-cucumber-workflow: - name: testdrive-cucumber-workflow (${{ matrix.configuration.cucumber }}, ${{ matrix.configuration.shard }}) - needs: cucumber-matrix - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: ${{ matrix.configuration.node }} - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ci-cucumber-fixtures - path: ${{ runner.temp }}/cucumber-fixtures - - name: Install Cucumber testdrive fixture dependencies - run: | - test "$(npm --version)" = '${{ matrix.configuration.npm }}' - echo 'Cucumber ${{ matrix.configuration.version }} covers: ${{ matrix.configuration.aliases }}' - mkdir -p "$RUNNER_TEMP/cucumber-testdrive" - cp "$RUNNER_TEMP/cucumber-fixtures/${{ matrix.configuration.fixture }}"/package*.json "$RUNNER_TEMP/cucumber-testdrive/" - npm ci --prefix "$RUNNER_TEMP/cucumber-testdrive" --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/cucumber-testdrive" --depth=0 - - name: Validate native scenarios, matching failures and source coverage - env: - DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/cucumber-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.configuration.shard }} - run: | - go test -v ./internal/testdrive -run '${{ matrix.configuration.tests }}' -failfast -count=1 -timeout=12m - - testdrive-webdriverio-workflow: - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # npm runs every scenario across shards; pnpm checks its layout on one. - # The latest lane also takes the latest tracer: support for a new - # WebdriverIO major lands there first. - include: - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '2/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '3/4'} - - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '4/4'} - - {package-manager: pnpm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/1'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '1/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '2/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '3/4'} - - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '4/4'} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - with: - go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: '24' - - name: Install WebdriverIO testdrive fixture dependencies - run: | - mkdir -p "$RUNNER_TEMP/wdio-testdrive" - printf '{"private":true}\n' > "$RUNNER_TEMP/wdio-testdrive/package.json" - if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - npm exec --yes --package=pnpm@10.20.0 -- pnpm --dir "$RUNNER_TEMP/wdio-testdrive" add --save-dev --save-exact @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} - else - npm install --prefix "$RUNNER_TEMP/wdio-testdrive" @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} --save-exact --no-audit --no-fund - fi - npm ls --prefix "$RUNNER_TEMP/wdio-testdrive" --depth=0 - - name: Validate real browser outcomes with Linux Xvfb supervision - env: - DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/wdio-testdrive/node_modules - DDTEST_CHROME_BINARY: /usr/bin/google-chrome - DDTEST_TEST_SHARD: ${{ matrix.shard }} - run: | - command -v xvfb-run - if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$/^passing$' -failfast -count=1 -timeout=8m - else - go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$' -failfast -count=1 -timeout=12m - fi - if [ '${{ matrix.shard }}' = '1/4' ] || [ '${{ matrix.package-manager }}' = 'pnpm' ]; then - go test -v ./internal/framework ./internal/testdrive -run '^TestWebdriverIO' -failfast -count=1 -timeout=5m - fi - - onboarding-evaluator-isolation: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.12" - - name: Build fake agent image (no model sessions) - run: | - scripts=skills/ddtest-onboarding-eval/scripts - docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" - echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" - - name: Verify environment, signals and detached process isolation - run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ada8e7a78..99f3e5aeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,274 +1,1239 @@ name: CI on: - push: - branches: [main] - pull_request: workflow_call: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: fast-ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -env: - PYTHONDONTWRITEBYTECODE: '1' - GORACE: atexit_sleep_ms=0 + push: + branches: + - "**" jobs: - plan: + test-binary: runs-on: ubuntu-latest - outputs: - go: ${{ steps.plan.outputs.go }} - ddto: ${{ steps.plan.outputs.ddto }} - evaluator: ${{ steps.plan.outputs.evaluator }} - integrations: ${{ steps.plan.outputs.integrations }} - playwright: ${{ steps.plan.outputs.playwright }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: - fetch-depth: 0 - - name: Select affected checks - id: plan + go-version-file: go.mod + - name: Build the CLI once for the public integration journeys env: - BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + CGO_ENABLED: '0' run: | - python3 scripts/ci_test_plan.py plan --base "$BASE" \ - --output "$RUNNER_TEMP/plan.json" --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" + mkdir -p "$RUNNER_TEMP/test-binary" + go build -o "$RUNNER_TEMP/test-binary/ddtest" . - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: pr-test-plan - path: ${{ runner.temp }}/plan.json + name: integration-cli + path: ${{ runner.temp }}/test-binary/ddtest + if-no-files-found: error - unit: - needs: plan - if: needs.plan.outputs.go == 'true' + test-support: runs-on: ubuntu-latest - timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: - fetch-depth: 0 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + go-version-file: go.mod + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + - name: Test onboarding evaluator + run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' + + # install.sh, and with the runner's pwsh install.ps1, against local release files. + - name: Test install scripts + run: python -m unittest discover -s scripts -p 'test_*.py' + + test-shards: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + # This job already has a compiler-specific cache below. Restoring + # setup-go's cache too collides with read-only module files. + cache: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - node-version: '24' - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - python-version: '3.12' - - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 with: - ruby-version: '3.4' - - name: Install runtime-backed unit test prerequisites + ruby-version: "3.4" + + - name: Cache Go modules + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + # ddto is its own module, checked by its own workflow; its benchmark + # fixtures are deliberately compact. + - name: Format check run: | - npm install --global yarn@1.22.22 - python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist - - name: Test affected Go packages and their dependents + unformatted="$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" + if [ -n "$unformatted" ]; then + echo "The following files are not formatted correctly:" + echo "$unformatted" + exit 1 + fi + + - name: Vet + run: go vet ./... + + - name: Select Go test shard + run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" + + - name: Configure Datadog Test Optimization + if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} + uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 + with: + languages: go + # Pin upstream Go 1.27 compatibility fixes until they reach a stable + # Orchestrion release. + go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 + api_key: ${{ secrets.DATADOG_API_KEY }} + site: datadoghq.com + service: ddtest + + - name: Run tests + run: go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... env: - BASE: ${{ github.event.pull_request.base.sha || github.event.before }} - DDTEST_REQUIRE_PROJECT_RUNTIMES: '1' - run: python3 scripts/ci_test_plan.py run --base "$BASE" --race + DD_ENV: ci + DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" + + - name: Retain shard coverage + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: unit-coverage-${{ matrix.shard }} + path: coverage.out + if-no-files-found: error + overwrite: true + retention-days: 1 - scripts: - needs: plan + # Keep the existing required check name; a skipped, cancelled or failed + # shard/support job must not turn the aggregate green. + test: + if: ${{ always() }} + needs: [test-shards, test-support] runs-on: ubuntu-latest - timeout-minutes: 10 + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} steps: + - name: Require every test job + env: + SHARDS_RESULT: ${{ needs.test-shards.result }} + SUPPORT_RESULT: ${{ needs.test-support.result }} + run: | + test "$SHARDS_RESULT" = success + test "$SUPPORT_RESULT" = success - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - python-version: '3.12' - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 - if: needs.plan.outputs.evaluator == 'true' + pattern: unit-coverage-* + path: ${{ runner.temp }}/unit-coverage + - name: Merge complete coverage + run: python3 scripts/go_test_shards.py merge --expected 3 --output coverage.out "$RUNNER_TEMP"/unit-coverage/*/coverage.out + - name: Upload coverage report to Datadog + if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} + uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 # v1.0.5 + with: + api_key: ${{ secrets.DATADOG_API_KEY }} + site: datadoghq.com + files: coverage.out + format: go-coverprofile + version: v5.19.0 + + # Dogfooding: runs the `test` suite with ddtest instrument, built from this + # checkout as a release builds it, instead of Orchestrion. Its results reach + # Test Optimization beside those of `test`, under this job's name. + test-ddtest-instrument-shards: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - if: needs.plan.outputs.evaluator == 'true' - with: - node-version: '22' - - name: Install evaluator package-manager fixtures - if: needs.plan.outputs.evaluator == 'true' - run: npm install --global yarn@1.22.22 pnpm@12.14.0 - - name: Test CI and install scripts - run: python3 -m unittest discover -s scripts -p 'test_*.py' - - name: Test changed onboarding evaluator - if: needs.plan.outputs.evaluator == 'true' - run: python3 -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p 'test_*.py' - - instrument: - needs: plan - if: needs.plan.outputs.go == 'true' + # This job already has a compiler-specific cache below. Restoring + # setup-go's cache too collides with read-only module files. + cache: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + # Its own key, so that its instrumented builds don't replace those of `test`. + - name: Cache Go modules + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Install Python test dependencies + run: python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist + + - name: Install evaluator package-manager fixture + run: npm install --global pnpm@12.14.0 + + - name: Select Go test shard + run: python scripts/go_test_shards.py select --shard "${{ matrix.shard }}/3" > "$RUNNER_TEMP/go-test-filter" + + # The release builder, as `make release` runs it for one target: ddtest + # with the runtime sources and tool wrapper that ddtest instrument provides. + - name: Build ddtest instrument from this checkout + run: | + version="dogfood-${GITHUB_SHA::12}" + go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ + -version "v0.0.0-$version" -ldflags "-X github.com/DataDog/ddtest/internal/buildinfo.Version=$version" \ + -target linux/amd64 -o "$RUNNER_TEMP/dist" + mkdir -p "$RUNNER_TEMP/bin" + mv "$RUNNER_TEMP/dist/ddtest-linux-amd64" "$RUNNER_TEMP/bin/ddtest" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + "$RUNNER_TEMP/bin/ddtest" instrument --version + + - name: Run tests + run: ddtest instrument go test -timeout=20m -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -run "$(cat "$RUNNER_TEMP/go-test-filter")" ./... + env: + DD_CIVISIBILITY_ENABLED: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' && 'parent' || 'false' }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_SERVICE: ddtest + DD_ENV: ci + JOB_CHECK_RUN_ID: ${{ job.check_run_id }} + DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" + # `test` uploads this commit's coverage report. + DD_CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED: "false" + + test-ddtest-instrument: + if: ${{ always() }} + needs: test-ddtest-instrument-shards runs-on: ubuntu-latest - timeout-minutes: 10 + steps: + - name: Require every instrumented shard + env: + SHARDS_RESULT: ${{ needs.test-ddtest-instrument-shards.result }} + run: test "$SHARDS_RESULT" = success + + rspec-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rspec: "3.12.0" + datadog_ci: "1.31.0" + - rspec: "3.13.2" + datadog_ci: "1.37.0" + steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Test RSpec adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install rspec --version "${{ matrix.rspec }}" --no-document + DDTEST_RSPEC_VERSION="${{ matrix.rspec }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestRSpecAdapterIntegration$' + DDTEST_RUBY_PLATFORM_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestRubyPlatformIntegration$' + + minitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - minitest: "5.26.0" + datadog_ci: "1.31.0" + - minitest: "6.0.6" + datadog_ci: "1.37.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 + with: + ruby-version: "3.4" + + - name: Test Minitest adapter + run: | + gem install datadog-ci --version "${{ matrix.datadog_ci }}" --no-document + gem install minitest --version "${{ matrix.minitest }}" --no-document + gem install rake --version "13.2.1" --no-document + DDTEST_MINITEST_VERSION="${{ matrix.minitest }}" \ + DDTEST_DATADOG_CI_VERSION="${{ matrix.datadog_ci }}" \ + go test -v ./internal/compatibility -run '^TestMinitestAdapterIntegration$' + + pytest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pytest: "8.4.2" + ddtrace: "4.11.0" + - pytest: "9.1.1" + ddtrace: "4.14.0" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Test pytest adapter + run: | + python -m pip install "ddtrace==${{ matrix.ddtrace }}" "pytest==${{ matrix.pytest }}" pytest-cov pytest-xdist + DDTEST_PYTHON_BINARY="$(command -v python)" \ + go test -v ./internal/compatibility -run '^TestPyTest(AdapterIntegration|SuiteModeSelectorBatches)$' + DDTEST_PYTHON_PLATFORM_INTEGRATION=1 \ + go test -v ./internal/compatibility -run '^TestPythonPlatformIntegration$' + DDTEST_PYTEST_PYTHON="$(command -v python)" \ + go test -v ./internal/testdrive -run '^TestPytest(RealValidation|CoverageExecutesOnlyInsideCopy|LoggingExecutesOnlyInsideCopy|RejectsEditableImportsFromOriginalCheckout|CoverageDataExecutesOnlyInsideCopy|IgnoredConfigurationUsesRoot|ObserverKeepsConcurrentDuplicateExecutions|PreflightReportsSelectedEnvironment)$' + + javascript-platform-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dd_trace: ["5.111.0", "6.15.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test JavaScript platform + run: | + dd_trace_dir="${RUNNER_TEMP}/dd-trace-${{ matrix.dd_trace }}" + npm install --prefix "${dd_trace_dir}" "dd-trace@${{ matrix.dd_trace }}" + DDTEST_DD_TRACE_NODE_MODULES="${dd_trace_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestJavaScript(Platform|ActionPreload)Integration$' + + jest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jest: ["27.5.1", "28.1.3", "29.4.3", "29.7.0", "30.5.1"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Jest adapter + run: | + jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" + npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" dd-trace@5.86.0 + DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestJest(Adapter|Testdrive)Integration$' + DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestJestCheckOnlyIntegration$' + + vitest-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + vitest: ["1.6.1", "2.1.9", "3.2.7", "4.1.11", "5.0.5"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Vitest adapter + run: | + vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" + # npm 10.9.8 crashes while resolving Vitest's peer dependencies. + # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. + npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "@vitest/coverage-v8@${{ matrix.vitest }}" "dd-trace@5.132.0" + DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ + DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' + DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestVitest(Configuration|ValidationArguments|MultiProjectRejectionKeepsConfigurationEvidence|RepeatedRunOptionIsNormalized)Integration$' + + mocha-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mocha: ["8.4.0", "9.2.2", "10.8.2", "11.7.6", "12.0.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Mocha adapter + run: | + mocha_dir="${RUNNER_TEMP}/mocha-${{ matrix.mocha }}" + npm install --prefix "${mocha_dir}" "mocha@${{ matrix.mocha }}" + DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestMocha.*Integration$' + DDTEST_MOCHA_NODE_MODULES="${mocha_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestMochaNativeObserverIntegration$' + + cypress-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cypress: ["12.17.4", "13.17.0", "14.5.4", "15.21.1", "16.0.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Cypress adapter + run: | + cypress_dir="${RUNNER_TEMP}/cypress-${{ matrix.cypress }}" + npm install --prefix "${cypress_dir}" "cypress@${{ matrix.cypress }}" + DDTEST_CYPRESS_BINARY="${cypress_dir}/node_modules/.bin/cypress" \ + DDTEST_CYPRESS_NODE_MODULES="${cypress_dir}/node_modules" \ + DDTEST_CYPRESS_XVFB="$(command -v xvfb-run)" \ + go test -v ./internal/compatibility -run '^TestCypressAdapter.*Integration$' + + playwright-compatibility: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + playwright: ["1.18.0", "1.31.1", "1.38.0", "1.59.1", "1.60.0", "1.62.1", "1.63.0"] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - name: Build release instrumentation and run its smoke packages + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Playwright adapter env: - DD_CIVISIBILITY_ENABLED: 'false' + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" run: | - go -C ddto run ./scripts/release -main .. -name ddtest -command instrument \ - -version v0.0.0-pr -target linux/amd64 -o "$RUNNER_TEMP/dist" - "$RUNNER_TEMP/dist/ddtest-linux-amd64" instrument go test -race ./internal/buildinfo ./internal/errcode + playwright_dir="${RUNNER_TEMP}/playwright-${{ matrix.playwright }}" + npm install --prefix "${playwright_dir}" "@playwright/test@${{ matrix.playwright }}" + DDTEST_PLAYWRIGHT_BINARY="${playwright_dir}/node_modules/.bin/playwright" \ + DDTEST_PLAYWRIGHT_NODE_MODULES="${playwright_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestPlaywrightAdapterIntegration$' - instrumentation-unit: - needs: plan - if: needs.plan.outputs.ddto == 'true' + cucumber-compatibility: runs-on: ubuntu-latest - timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + cucumber: ["7.0.0", "8.11.1", "9.6.0", "10.9.0", "11.3.0", "12.9.0", "13.2.1"] + steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - cache-dependency-path: ddto/go.mod - - name: Audit and test instrumentation units on the current toolchain - working-directory: ddto - run: | - python3 scripts/upstream.py verify - python3 scripts/dependency_boundary.py - python3 -m unittest discover -s scripts -p 'test_*.py' - go vet ./... - packages=$(go list ./... | grep -v '/integration$') - go test -short -race $packages - - integration: - needs: plan - if: needs.plan.outputs.integrations != '[]' + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Test Cucumber adapter + run: | + cucumber_dir="${RUNNER_TEMP}/cucumber-${{ matrix.cucumber }}" + npm install --prefix "${cucumber_dir}" "@cucumber/cucumber@${{ matrix.cucumber }}" + DDTEST_CUCUMBER_BINARY="${cucumber_dir}/node_modules/.bin/cucumber-js" \ + DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ + DDTEST_CUCUMBER_VERSION="${{ matrix.cucumber }}" \ + go test -v ./internal/compatibility -run '^TestCucumberAdapterIntegration$' + DDTEST_CUCUMBER_NODE_MODULES="${cucumber_dir}/node_modules" \ + go test -v ./internal/testdrive -run '^TestCucumberSupportMetadataIntegration$' + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + + - name: golangci-lint + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 + with: + version-file: .golangci-lint-version + args: --timeout=5m + + jest-testdrive-workflow: + needs: test-binary + name: jest-testdrive-workflow (${{ matrix.node }}, ${{ matrix.jest }}, ${{ matrix.group }} ${{ matrix.shard }}) runs-on: ubuntu-latest - timeout-minutes: 25 strategy: fail-fast: false matrix: - configuration: ${{ fromJSON(needs.plan.outputs.integrations) }} - name: smoke (${{ matrix.configuration.id }}) + # Per runtime: TestPublicJestTestdrive over two shards, plus two groups of + # configuration tests of similar length; together they are the full set. + include: + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} + - {node: '24', jest: '30.5.1', tracer: '6.15.0', typescript: '', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} + # Unpinned: catches framework releases our pinned versions predate. + - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '24', jest: 'latest', tracer: '6.15.0', typescript: '', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '1/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: testdrive, shard: '2/2', tests: 'TestPublicJestTestdrive'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: packages, shard: '1/1', tests: 'TestPublicJest(PackageConfiguration|ForwardedConfiguration)|TestPublicCoordinatorJestFactsAndFreshReport'} + - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - if: matrix.configuration.runtime == 'node' with: - node-version: '24' + node-version: ${{ matrix.node }} + - name: Reuse the locked Jest and tracer fixture + uses: ./.github/actions/node-test-fixture + with: + directory: ${{ runner.temp }}/jest-testdrive + dependencies: jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 + - name: Validate paired outcomes, all feature probes and cleanup + env: + DDTEST_JEST_NODE_MODULES: ${{ runner.temp }}/jest-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(${{ matrix.tests }})$' -count=1 -timeout=12m + + windows-probe-diagnostics: + runs-on: windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - name: Test Windows probe filesystem diagnostics + run: go test ./internal/testdrive -run '^(TestProbePublicationExplainsWindowsFilesystemErrors|TestReportsCanReplaceExistingDestinationTwice)$' -count=1 + + + testdrive-language-workflow: + needs: test-binary + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + framework: [pytest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - if: matrix.configuration.runtime == 'python' with: python-version: '3.12' + - name: Run Python testdrive + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + run: go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives/${{ matrix.framework }}$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: testdrive-${{ matrix.framework }} + path: ${{ runner.temp }}/ddtest-workflow-reports + + ruby-testdrive: + needs: test-binary + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - name: pinned-minitest-6 + datadog_ci: "1.40.0" + rspec: "3.13.2" + minitest: "6.0.6" + rake: "13.4.2" + - name: pinned-minitest-5 + datadog_ci: "1.40.0" + rspec: "3.13.2" + minitest: "5.27.0" + rake: "13.4.2" + - name: latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1 - if: matrix.configuration.runtime == 'ruby' with: ruby-version: '3.4' + - name: Install Ruby validation dependencies + env: + DATADOG_CI: ${{ matrix.datadog_ci }} + RSPEC: ${{ matrix.rspec }} + MINITEST: ${{ matrix.minitest }} + RAKE: ${{ matrix.rake }} + run: | + install() { if [ -n "$2" ]; then gem install "$1" --version "$2" --no-document; else gem install "$1" --no-document; fi; } + install datadog-ci "$DATADOG_CI" + install rspec "$RSPEC" + install minitest "$MINITEST" + install rake "$RAKE" + gem install activesupport --conservative --no-document + gem list | grep -E '^(datadog-ci|rspec-core|minitest|rake|activesupport) ' + - name: Run Ruby testdrive integration tests + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_RUN_RUBY_INTEGRATION_TEST: '1' + DDTEST_RUN_RUBY_FORK_INTEGRATION_TEST: '1' + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + run: | + go test -v ./internal/testdrive -run '^TestPublicPythonRubyTestdrives$/^(rspec|minitest)$' -count=1 -timeout=20m + go test -v ./internal/testdrive -run '^TestRuby.+Integration$' -skip '^TestRubyValidationIntegration$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ruby-testdrive-${{ matrix.name }} + path: ${{ runner.temp }}/ddtest-workflow-reports + if-no-files-found: ignore + retention-days: 7 + + java-testdrive: + needs: test-binary + name: java-testdrive (${{ matrix.name }}, ${{ matrix.java }}, ${{ matrix.tracer }}, ${{ matrix.shard }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # Each scenario runs a full paired validation with real Maven or Gradle builds; + # the scenarios are spread over shards (DDTEST_TEST_SHARD). + name: [pinned, latest] + shard: ['1/4', '2/4', '3/4', '4/4'] + include: + - name: pinned + java: '17' + tracer: '1.67.1' + - name: latest + java: '21' + tracer: latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 - if: matrix.configuration.runtime == 'java' || matrix.configuration.runtime == 'gradle' with: distribution: temurin - java-version: '17' + java-version: ${{ matrix.java }} + # The Maven fixtures are generated by the test, so the repository has no + # pom.xml for setup-java's cache key; key the cache on the test instead. + - name: Cache Maven dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ matrix.java }}-${{ hashFiles('internal/testdrive/java_validation_integration_test.go') }} + restore-keys: | + ${{ runner.os }}-maven-${{ matrix.java }}- + # The Gradle fixtures run the gradle on PATH, as a CI job with this + # action does; it also caches the Gradle user home. - uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 - if: matrix.configuration.runtime == 'gradle' with: gradle-version: '8.10.2' - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 - if: matrix.configuration.runtime == 'node' + - name: Run Java testdrive integration tests + env: + DDTEST_RUN_JAVA_INTEGRATION_TEST: '1' + DDTEST_JAVA_TRACER_VERSION: ${{ matrix.tracer }} + DDTEST_FRAMEWORK_REPORTS: ${{ runner.temp }}/ddtest-workflow-reports + DDTEST_TEST_SHARD: ${{ matrix.shard }} + # Every validation starts Maven many times; short-lived Maven JVMs + # start faster without the optimizing compiler. Both runs inherit it. + MAVEN_OPTS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 + run: go test -v ./internal/testdrive -run '^TestJavaValidationIntegration$' -count=1 -timeout=20m + - name: Retain failed testdrive reports + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - path: | - ~/.npm - ~/.cache/Cypress - key: smoke-${{ runner.os }}-${{ matrix.configuration.id }}-${{ hashFiles('scripts/ci_test_plan.json') }} - restore-keys: smoke-${{ runner.os }}-${{ matrix.configuration.id }}- - - name: Run selected smoke and require every named scenario - run: python3 scripts/ci_smoke.py --id '${{ matrix.configuration.id }}' - - playwright: - needs: plan - if: needs.plan.outputs.playwright == 'true' + name: java-testdrive-${{ matrix.name }}-${{ strategy.job-index }} + path: ${{ runner.temp }}/ddtest-workflow-reports + if-no-files-found: ignore + retention-days: 7 + + testdrive-vitest-workflow: + needs: test-binary runs-on: ubuntu-latest - timeout-minutes: 17 - container: - image: mcr.microsoft.com/playwright:v1.55.1-noble + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - vitest: '3.2.7' + tracer: '5.132.0' + - vitest: '4.1.11' + tracer: '6.21.0' + - vitest: '5.0.5' + tracer: '6.21.0' + - vitest: 'latest' + tracer: '6.21.0' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' - - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - run: python3 scripts/ci_smoke.py --id playwright + - name: Resolve Vitest version + run: | + version=$(npm view "vitest@${{ matrix.vitest }}" version) + echo "Vitest $version" + echo "DDTEST_VITEST_VERSION=$version" >> "$GITHUB_ENV" + - name: Validate Vitest CLI and Node API paths against local intake + env: + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST: '1' + DDTEST_VITEST_TRACER_VERSION: ${{ matrix.tracer }} + run: go test -v ./internal/testdrive -run '^TestPublicVitestRunnerTestdrive$' -count=1 -timeout=15m - evaluator-isolation: - needs: plan - if: needs.plan.outputs.evaluator == 'true' + onboard-apply-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - persist-credentials: false - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: - python-version: '3.12' - - name: Build offline evaluator image + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Validate generated workflow edits with actionlint run: | - scripts=skills/ddtest-onboarding-eval/scripts - docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" - echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" - - run: python3 -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v + go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/onboard -run '^TestGeneratedSetupWorkflow.*Actionlint$' -count=1 + - name: Install apply fixture dependencies + run: npm install --prefix "$RUNNER_TEMP/apply-testdrive" vitest@4.1.11 mocha@11.7.6 nyc@17.1.0 dd-trace@6.20.1 @cucumber/cucumber@12.2.0 @wdio/cli@9.20.0 @wdio/local-runner@9.20.0 @wdio/mocha-framework@9.20.0 --no-audit --no-fund + - name: Apply setup and validate against local intake + env: + DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_VITEST_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/apply-testdrive/node_modules + DDTEST_CHROME_BINARY: /usr/bin/google-chrome + run: DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/testdrive -run '^TestPublic(OnboardApply|MochaApply|WebdriverIOApply|CucumberApply)$' -count=1 -timeout=8m - lint: - needs: plan - if: needs.plan.outputs.go == 'true' + testdrive-mocha-workflow: + needs: test-binary runs-on: ubuntu-latest + timeout-minutes: 12 + strategy: + fail-fast: false + matrix: + mocha: ['11.7.6', 'latest'] + shard: ['1/2', '2/2'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - - name: Format check - run: test -z "$(git ls-files -z '*.go' ':!:ddto/**' | xargs -0 gofmt -s -l)" - - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - version-file: .golangci-lint-version - args: --timeout=5m + node-version: '24' + - name: Install Mocha testdrive fixture dependencies + run: | + npm install --prefix "$RUNNER_TEMP/mocha-testdrive" "mocha@${{ matrix.mocha }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.21.0 --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/mocha-testdrive" --depth=0 + - name: Validate CLI, programmatic runner, existing failures and coverage gate + env: + DDTEST_MOCHA_NODE_MODULES: ${{ runner.temp }}/mocha-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(TestPublicMochaTestdrive|TestMochaProbeCoverageThresholdControls|TestMochaDelayedCommandUsesSharedTimeout|TestMochaPreflightResolvesSelectedPackage)$' -count=1 -timeout=10m - # Preserve the required check name and reject every failed or unexpectedly - # skipped selected job. Only the planner may intentionally omit a check. - test: - if: ${{ always() }} - needs: [plan, unit, scripts, instrument, instrumentation-unit, integration, playwright, evaluator-isolation, lint] + playwright-latest: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + steps: + - id: resolve + run: echo "version=$(npm view @playwright/test version)" >> "$GITHUB_OUTPUT" + + testdrive-playwright-workflow: + needs: [test-binary, playwright-latest] + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + playwright: ['1.55.1', 'latest'] + shard: ['1/2', '2/2'] + # The official image ships the browsers and their system packages for this + # exact Playwright release, so nothing is downloaded or installed per run. + container: + image: mcr.microsoft.com/playwright:v${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }}-noble + env: + PLAYWRIGHT_VERSION: ${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + # The workspace is owned by the runner user, not the container's root; + # go build reads it with git for VCS stamping. + - name: Trust the workspace for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Isolate Playwright fixture caches + run: echo "npm_config_cache=$RUNNER_TEMP/playwright-testdrive/cache" >> "$GITHUB_ENV" + - name: Install Playwright testdrive fixture dependencies + run: | + npm install --prefix "$RUNNER_TEMP/playwright-testdrive" "@playwright/test@$PLAYWRIGHT_VERSION" dd-trace@6.20.1 --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/playwright-testdrive" --depth=0 + - name: Validate browser execution, existing failures and supervised cleanup + env: + DDTEST_PLAYWRIGHT_NODE_MODULES: ${{ runner.temp }}/playwright-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: go test -v ./internal/testdrive -run '^(TestPublicPlaywrightTestdrive|TestPlaywrightDiscoveryOwnsImportSideEffects|TestPlaywrightDiscoveryKeepsConfiguredReportsOutOfTheCheckout)$' -count=1 -timeout=12m + + testdrive-cypress-workflow: + needs: test-binary runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Scenarios are split by integration_shard_test.go; each runs exactly + # once per Cypress version. + cypress: ['15.1.0', 'latest'] + shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] steps: - - name: Require every selected check + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve Cypress version + id: cypress + run: echo "version=$(npm view "cypress@${{ matrix.cypress }}" version | tail -n 1)" >> "$GITHUB_OUTPUT" + # The Cypress binary downloads during npm install; reuse it per release. + - name: Cache Cypress binary + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/cypress-testdrive-cache + key: ${{ runner.os }}-cypress-binary-${{ steps.cypress.outputs.version }} + - name: Install Cypress testdrive fixture dependencies env: - RESULTS: ${{ toJSON(needs) }} - run: | - python3 - <<'PY' - import json, os - results = json.loads(os.environ['RESULTS']) - if results['plan']['result'] != 'success': - raise SystemExit('Test selection failed') - plan = results['plan']['outputs'] - required = {'plan', 'scripts'} - if plan['go'] == 'true': - required.update(['unit', 'instrument', 'lint']) - if plan['ddto'] == 'true': - required.add('instrumentation-unit') - if plan['evaluator'] == 'true': - required.add('evaluator-isolation') - if json.loads(plan['integrations']): - required.add('integration') - if plan['playwright'] == 'true': - required.add('playwright') - bad = {name: job['result'] for name, job in results.items() - if job['result'] != ('success' if name in required else 'skipped')} - if bad: - raise SystemExit(f'Selected checks incomplete: {bad}') - PY + CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache + run: | + npm install --prefix "$RUNNER_TEMP/cypress-testdrive" "cypress@${{ matrix.cypress }}" dd-trace@6.20.1 typescript@5.9.3 --save-exact --no-audit --no-fund + npm ls --prefix "$RUNNER_TEMP/cypress-testdrive" --depth=0 + - name: Validate original CLI, action API and matching failures with TIA deferred + env: + DDTEST_CYPRESS_NODE_MODULES: ${{ runner.temp }}/cypress-testdrive/node_modules + CYPRESS_CACHE_FOLDER: ${{ runner.temp }}/cypress-testdrive-cache + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: xvfb-run -a go test -v ./internal/testdrive -run '^(TestPublicCypressTestdrive|TestCypressTypeScriptWrapperTranspilesToCommonJS)$' -failfast -count=1 -timeout=12m + + # Resolve floating dependencies before expanding the matrix. Identical aliases + # share locked fixtures and jobs within this comprehensive run. + cucumber-matrix: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve versions, lock fixtures and select distinct configurations + id: plan + run: | + python3 scripts/ci_cucumber_matrix.py --output "$RUNNER_TEMP/cucumber-fixtures" \ + --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures + if-no-files-found: error + retention-days: 14 + + testdrive-cucumber-workflow: + name: testdrive-cucumber-workflow (${{ matrix.configuration.cucumber }}, ${{ matrix.configuration.shard }}) + needs: [cucumber-matrix, test-binary] + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.configuration.node }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures + - name: Install Cucumber testdrive fixture dependencies + run: | + test "$(npm --version)" = '${{ matrix.configuration.npm }}' + echo 'Cucumber ${{ matrix.configuration.version }} covers: ${{ matrix.configuration.aliases }}' + mkdir -p "$RUNNER_TEMP/cucumber-testdrive" + cp "$RUNNER_TEMP/cucumber-fixtures/${{ matrix.configuration.fixture }}"/package*.json "$RUNNER_TEMP/cucumber-testdrive/" + - name: Reuse the locked Cucumber fixture + uses: ./.github/actions/node-test-fixture + with: + directory: ${{ runner.temp }}/cucumber-testdrive + - name: Validate native scenarios, matching failures and source coverage + env: + DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/cucumber-testdrive/node_modules + DDTEST_TEST_SHARD: ${{ matrix.configuration.shard }} + run: | + go test -v ./internal/testdrive -run '${{ matrix.configuration.tests }}' -failfast -count=1 -timeout=12m + + testdrive-webdriverio-workflow: + needs: test-binary + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # npm runs every scenario across shards; pnpm checks its layout on one. + # The latest lane also takes the latest tracer: support for a new + # WebdriverIO major lands there first. + include: + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '2/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '3/4'} + - {package-manager: npm, wdio: '9.20.0', tracer: '6.20.1', shard: '4/4'} + - {package-manager: pnpm, wdio: '9.20.0', tracer: '6.20.1', shard: '1/1'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '1/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '2/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '3/4'} + - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '4/4'} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Install WebdriverIO testdrive fixture dependencies + run: | + mkdir -p "$RUNNER_TEMP/wdio-testdrive" + printf '{"private":true}\n' > "$RUNNER_TEMP/wdio-testdrive/package.json" + if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + npm exec --yes --package=pnpm@10.20.0 -- pnpm --dir "$RUNNER_TEMP/wdio-testdrive" add --save-dev --save-exact @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} + else + npm install --prefix "$RUNNER_TEMP/wdio-testdrive" @wdio/cli@${{ matrix.wdio }} @wdio/local-runner@${{ matrix.wdio }} @wdio/mocha-framework@${{ matrix.wdio }} @wdio/jasmine-framework@${{ matrix.wdio }} @wdio/spec-reporter@${{ matrix.wdio }} dd-trace@${{ matrix.tracer }} --save-exact --no-audit --no-fund + fi + npm ls --prefix "$RUNNER_TEMP/wdio-testdrive" --depth=0 + - name: Validate real browser outcomes with Linux Xvfb supervision + env: + DDTEST_WEBDRIVERIO_NODE_MODULES: ${{ runner.temp }}/wdio-testdrive/node_modules + DDTEST_CHROME_BINARY: /usr/bin/google-chrome + DDTEST_TEST_SHARD: ${{ matrix.shard }} + run: | + command -v xvfb-run + if [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$/^passing$' -failfast -count=1 -timeout=8m + else + go test -v ./internal/testdrive -run '^TestPublicWebdriverIOTestdrive$' -failfast -count=1 -timeout=12m + fi + if [ '${{ matrix.shard }}' = '1/4' ] || [ '${{ matrix.package-manager }}' = 'pnpm' ]; then + go test -v ./internal/framework ./internal/testdrive -run '^TestWebdriverIO' -failfast -count=1 -timeout=5m + fi + + onboarding-evaluator-isolation: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - name: Build fake agent image (no model sessions) + run: | + scripts=skills/ddtest-onboarding-eval/scripts + docker build --target offline -t ddtest-eval-offline -f "$scripts/Dockerfile" "$scripts" + echo "DDTEST_EVAL_TEST_IMAGE=$(docker image inspect --format '{{.Id}}' ddtest-eval-offline)" >> "$GITHUB_ENV" + - name: Verify environment, signals and detached process isolation + run: python -m unittest discover -s skills/ddtest-onboarding-eval/scripts -p test_container.py -v diff --git a/.github/workflows/go-instrumentation.yml b/.github/workflows/go-instrumentation.yml index 2b070016f..1df8ba4ea 100644 --- a/.github/workflows/go-instrumentation.yml +++ b/.github/workflows/go-instrumentation.yml @@ -1,14 +1,19 @@ name: Go instrumentation # The Go instrumentation that ddtest instrument runs is its own module in ddto/. -# Its complete matrix runs for releases, nightly upstream checks and manual -# dispatch. Pull requests use the native unit checks in ci.yml instead. +# Its suite runs from there, with ddtest's executable as the driver. Every pull +# request and push to main runs this workflow, so that its result can be a +# required check: when a run already verified the files that the suite depends +# on (see the plan job's inputs), the groups do not run again. # -# Manual runs use the dd-trace-go and Orchestrion versions in -# ddto/scripts/supported_versions.json by default. The nightly run tests the -# versions that the module proxy has then, and fails when that file is out of -# date. The release workflow calls this one with compatibility: full: those +# Pull requests and pushes test the dd-trace-go and Orchestrion versions in +# ddto/scripts/supported_versions.json. The nightly run tests the versions that +# the module proxy has then, and fails when that file is out of date. The +# release workflow calls this one with compatibility: full first: those # versions, each one run even when another has the same edited SDK packages. on: + push: + branches: [main] + pull_request: schedule: - cron: '17 3 * * *' workflow_call: @@ -26,7 +31,8 @@ on: default: recorded permissions: contents: read -# Independent release, nightly and manual runs never share a group. +# A new push to a pull request replaces its running matrix. Other events never +# share a group, so main and scheduled runs always finish. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23e49da0d..155053f97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,18 +43,29 @@ jobs: GH_TOKEN: ${{ steps.octo-sts.outputs.token }} run: bash .github/scripts/release.sh validate - # Every release waits for the complete framework, platform and Go instrumentation - # matrices, including every currently supported SDK and Orchestrion version. + ci: + needs: authorize + uses: ./.github/workflows/ci.yml + permissions: + contents: read + actions: read + secrets: inherit + + # Before a release, the Go instrumentation suite runs every dd-trace-go and + # Orchestrion version that ddtest instrument supports, as the module proxy + # has them now, so that no release ships without them passing. compatibility: needs: authorize - uses: ./.github/workflows/ci-full.yml + uses: ./.github/workflows/go-instrumentation.yml + with: + compatibility: full permissions: contents: read actions: read secrets: inherit release: - needs: compatibility + needs: [ci, compatibility] if: github.repository == 'DataDog/ddtest' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: diff --git a/Makefile b/Makefile index 90979fb73..f239804de 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := build -.PHONY: clean fmt vet lint test test-fast build install run release +.PHONY: clean fmt vet lint test build install run release VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) GOBIN ?= $(shell go env GOBIN) @@ -16,8 +16,6 @@ lint: vet golangci-lint run --timeout=5m test: go test ./... -test-fast: - python3 scripts/ci_test_plan.py run --base "$(or $(TEST_BASE),origin/main)" build: test lint go build -ldflags="$(LDFLAGS)" -o ddtest main.go install: diff --git a/ddto/AGENTS.md b/ddto/AGENTS.md index 7b697da1f..45484a21c 100644 --- a/ddto/AGENTS.md +++ b/ddto/AGENTS.md @@ -360,7 +360,7 @@ maintenance guides. Before proposing a change: - `docs/benchmarks.md` and `docs/results/**` are generated by the benchmark scripts; never edit them by hand. See [the benchmark guide](docs/build-benchmarks.md). -- Pull requests run native units on the current Go toolchain and a release - binary smoke. The full compatibility matrix gates releases and runs nightly; - it is also available by manual dispatch +- CI reuses a passed run when a change leaves the suite's inputs unchanged, + such as one limited to `docs/` and `README.md`; other changes run the full + compatibility matrix ([details](docs/validation.md#go-instrumentation-workflow)). diff --git a/ddto/docs/validation.md b/ddto/docs/validation.md index 2d50f7f65..c3b0d32d5 100644 --- a/ddto/docs/validation.md +++ b/ddto/docs/validation.md @@ -207,7 +207,7 @@ families and tip: | Linux | 1.25 | Native Mini suite in normal and `-race` modes; `GOTOOLCHAIN=local` prevents an upgrade | | Linux | 1.26 | Complete SDK differential suite | | Linux | 1.27 | Complete SDK differential suite in normal and `-race` modes | -| Linux | tip | Native Mini suite and manual SDK span-copy cases; source commit recorded before building Go | +| Linux | tip | Native Mini suite and manual SDK span-copy cases; source commit recorded before building Go. Pull requests, nightly and manual dispatch | | macOS | 1.27 | Normal suite | | Windows | 1.27 | Normal suite | @@ -235,22 +235,21 @@ report step requires the independent Orchestrion reference and exports feature outcomes, counts and timings. Logs and JSON reports are uploaded even when a test fails. -The full matrix runs nightly, on manual dispatch, and as part of the -[Full compatibility](../../.github/workflows/ci-full.yml) gate before a release. -Release builds wait for its complete compatibility evidence on the same commit. -Manual Go instrumentation runs default to recorded SDK/Orchestrion versions; -`compatibility: full` and release runs resolve every currently supported version. - -Pull requests and pushes to `main` use the smaller native checks in -[CI](../../.github/workflows/ci.yml). When instrumentation inputs change, these -audit the sources and dependency boundary, vet the module and run its local -packages with `-short -race` on the current toolchain. Code changes also build -the release executable and instrument two small packages. They omit the -integration package, SDK differential comparisons, other OS/toolchain versions, -Go tip and aggregate coverage. Documentation changes omit runtime checks. -[PR selection](../../docs/ci-test-selection.md) describes the active policy and -the contributor command. A green PR validates this smaller contract; release -and nightly runs provide comprehensive compatibility evidence. +Every pull request and push to `main`, a nightly schedule and manual dispatch +start the workflow, so that its last job, Complete compatibility evidence, can +be a required check. A new push to a pull request cancels its previous run. The +suite depends on this directory, except `docs/` and `README.md`, and on what +builds and runs ddtest's executable: ddtest's `main.go`, `go.mod`, `go.sum`, +`internal/cmd/instrument.go`, install scripts and the workflow. A pull request +or a push to `main` whose copies of those files already passed a complete run of +a branch in this repository reuses that run instead of repeating the matrix: +typically a pull request that changes none of them, or the squash merge of an +up-to-date pull request. It uploads that run's coverage of ddto's packages to +Datadog for its own commit, since Datadog counts lines without a report as +uncovered. Nightly and manual runs always run the matrix. Pull requests, nightly +and manual runs also test Go tip, which changes independently of this +repository: a pull request can fail on tip without causing it. A push to `main` +that runs the matrix itself skips tip. ### Parallel test groups diff --git a/docs/ci-performance.md b/docs/ci-performance.md new file mode 100644 index 000000000..ce1039f63 --- /dev/null +++ b/docs/ci-performance.md @@ -0,0 +1,49 @@ +# CI fixture reuse + +PR branches, main and releases use the same CI workflow and the same onboard and +testdrive scenarios. Both existing Go instrumentation paths and their coverage +remain enabled. Datadog Test Impact Analysis owns test skipping; this workflow +does not maintain a separate changed-file test selector. + +## Shared CLI executable + +The `test-binary` job builds one static executable for the Linux public CLI journeys, +including jobs in browser containers. +Every framework job downloads that run's `integration-cli` artifact and sets +`DDTEST_TEST_BINARY` to its absolute path. The Go test helper uses that executable +across temporary customer checkouts. Outside CI, it builds the CLI once per test +process; the test process's isolated temporary directory owns its cleanup. + +## Locked Node fixtures + +The local `node-test-fixture` action prepares a lockfile and reuses installed +dependencies only when the OS, architecture, exact Node/npm versions and complete +lockfile match. Cache misses run `npm ci`; there are no partial restore keys. +Dependencies are saved before tests run, keeping runtime-generated caches out of +the reusable fixture. +Jest matrix jobs and Cucumber workers use this action. Browser fixtures with +additional external installation state retain their existing setup. + +The Cucumber planner resolves `latest` once before expanding the matrix and +creates one lockfile per resolved version. Labels share a job only when the +complete dependency graph, runtime, test filter and shard are identical. All +distinct compatibility cases remain. When `latest` resolves to pinned `13.3.0`, +13 requested lanes become 9 executions; a distinct latest version keeps its lanes. + +Inspect the resolved configurations with Node 24 and npm registry access: + +```sh +python3 scripts/ci_cucumber_matrix.py --output /tmp/cucumber-ci-fixtures \ + --summary /tmp/cucumber-ci-matrix.md +``` + +The output directory must not exist. Inspect `matrix.json`, the summary and the +version directories' lockfiles; remove the directory afterwards. + +## Python prerequisites + +The Python version-check subprocess disables Datadog reporting and telemetry; +it only imports dependencies and checks their versions. The actual testdrive +journeys keep their original reporting, local intake, configuration variants and +six feature assertions. No validation scenario is removed or moved to a release +only workflow. diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md deleted file mode 100644 index fb0e24fe8..000000000 --- a/docs/ci-test-selection.md +++ /dev/null @@ -1,106 +0,0 @@ -# Faster PR and contributor checks - -The `CI` workflow actively selects checks for pull requests and pushes to `main`. -It runs native Go tests once, without global coverage or SDK weaving. Affected -frameworks get a pinned Linux smoke, rather than the complete historical-version -and scenario matrices. This is the PR acceptance policy, with no shadow phase. - -`Full compatibility` retains the complete framework matrices, both instrumented -root Go test paths, coverage, and the Go instrumentation OS/toolchain/SDK matrix. -Every release waits for it on the commit being released. Maintainers can also -run it from Actions → Full compatibility → Run workflow on any branch. The -existing nightly Go instrumentation matrix continues to catch upstream changes. -The full framework matrix has no recurring schedule; it runs for releases and -manual dispatches. - -## Local feedback - -From the repository root, with Python 3 and the Go version in `go.mod`: - -```sh -make test-fast # compare with the merge base of origin/main -make test-fast TEST_BASE= # choose another base -``` - -This includes staged, unstaged, deleted, renamed and non-ignored untracked files. -A production Go edit selects its package and every transitive importer, including -imports made only by tests. A test-only edit selects its own package. Non-Go -inputs (including embedded assets), removed packages and unknown history select -all root packages. Go's own build and test caches remain enabled. `make test` -still runs the whole root module. The instrumentation in `ddto/` is a separate -module; its local checks are described in [its guide](../ddto/AGENTS.md#changing-this-repository). - -Inspect the same integration selection that CI uses: - -```sh -python3 scripts/ci_test_plan.py plan --base origin/main --output /tmp/pr-plan.json -cat /tmp/pr-plan.json -``` - -To execute a selected framework smoke, install its runtime and dependencies -listed in `scripts/ci_test_plan.json`, then run, for example: - -```sh -python3 scripts/ci_smoke.py --id cucumber -``` - -The smoke command installs its npm fixture in a temporary directory and removes -it afterwards. Ruby smokes install the listed gems in the active Ruby environment. -Browser checks need the CI browser/Xvfb environment; Playwright uses the pinned -official image in `ci.yml`. These checks use controlled local fixtures without -Datadog credentials. - -## Selection and acceptance - -[`scripts/ci_test_plan.json`](../scripts/ci_test_plan.json) owns framework paths, -embedded instructions/scripts, JavaScript dependencies and smoke recipes. -Python-only changes select Python; JavaScript platform changes also select all -JavaScript frameworks. Shared or unclassified production inputs select every -PR smoke. Instrumentation module changes select its units and the root/release -smoke without unrelated framework integrations. Pure unit-test edits omit -external smokes. Integration tests and ordinary `_test.go` files registered in -`runtime_test_inputs` select their framework smokes. Register runtime-backed test -files there when adding tests that need framework fixtures beyond the unit job. -The native Go job installs Python/pytest/ddtrace, Ruby/Bundler and Node/Yarn, -and sets `DDTEST_REQUIRE_PROJECT_RUNTIMES=1` so focused runtime regressions in -selected packages fail on missing prerequisites instead of silently skipping. -Documentation-only changes omit runtime jobs; evaluator-only changes run the -evaluator checks. The small Python suite tests CI and installer scripts on every -PR. Missing base history selects all PR checks, never an empty plan. - -Most framework smokes exercise a passing customer fixture and an existing -failure. Python, Ruby, Java and Vitest use their existing validation journeys; -Java covers both Maven and Gradle. PRs use one pinned framework/tracer/runtime -combination per journey. Older versions, `latest`, remaining scenarios, SDK parity -and platform differences are checked by Full compatibility. -Each smoke retains its full compatibility lane's Go test timeout, with at least -five additional minutes in the job limit for setup and compilation. - -The `pr-test-plan` artifact and job summary show the selection. Every smoke -requires explicit named Go test cases to report `pass`: a filtered, missing or -skipped case fails. The required `test` check rejects failed, cancelled or -unexpectedly skipped selected jobs. The existing required `lint` check remains. -New production paths default to all smokes; add narrower ownership only when its -framework boundary is known and tested. - -## Deduplicating resolved Cucumber configurations - -The full suite resolves `latest` once and creates a dependency lockfile per exact -Cucumber version before expanding its jobs. Requested labels share a job only -when the resolved version, complete dependency graph, exact Node/npm versions, -test filter and shard all match. For example, when `latest` resolves to the -pinned `13.3.0`, its four duplicate jobs collapse, reducing 13 requested lanes -to 9. A new latest release creates separate jobs again. The `8.11.1` floor smoke -remains distinct from full scenario lanes. - -Workers download the planned fixture, verify npm's version and use `npm ci`. -Registry or lockfile failures fail planning. To inspect a live resolution with -Node 24, npm and registry access, use a fresh directory: - -```sh -python3 scripts/ci_cucumber_matrix.py --output /tmp/cucumber-ci-fixtures \ - --summary /tmp/cucumber-ci-matrix.md -``` - -Inspect `matrix.json`, the summary and each version's `package-lock.json`. The -summary lists all requested aliases represented by each remaining job. diff --git a/internal/releaseworkflow/ci_shards_test.go b/internal/releaseworkflow/ci_shards_test.go index 4ff48d3c6..1d357ed66 100644 --- a/internal/releaseworkflow/ci_shards_test.go +++ b/internal/releaseworkflow/ci_shards_test.go @@ -14,6 +14,7 @@ import ( type ciTestJob struct { If string Needs any + Uses string Strategy struct { FailFast bool `yaml:"fail-fast"` Matrix struct{ Shard []any } @@ -24,9 +25,43 @@ type ciTestJob struct { } } +func TestReleaseUsesTheSameCIAndRetainsTheInstrumentationGate(t *testing.T) { + data, err := os.ReadFile("../../.github/workflows/release.yml") + require.NoError(t, err) + var workflow struct{ Jobs map[string]ciTestJob } + require.NoError(t, yaml.Unmarshal(data, &workflow)) + require.Equal(t, "./.github/workflows/ci.yml", workflow.Jobs["ci"].Uses) + require.Equal(t, "./.github/workflows/go-instrumentation.yml", workflow.Jobs["compatibility"].Uses) + require.Equal(t, []any{"ci", "compatibility"}, workflow.Jobs["release"].Needs) +} + +func TestPublicCLIJobsShareOneExecutable(t *testing.T) { + jobs := ciTestJobs(t) + for _, name := range []string{"jest-testdrive-workflow", "testdrive-language-workflow", "ruby-testdrive", + "java-testdrive", "testdrive-vitest-workflow", "onboard-apply-workflow", "testdrive-mocha-workflow", + "testdrive-playwright-workflow", "testdrive-cypress-workflow", "testdrive-cucumber-workflow", "testdrive-webdriverio-workflow"} { + job := jobs[name] + switch dependencies := job.Needs.(type) { + case string: + require.Equal(t, "test-binary", dependencies, name) + case []any: + require.Contains(t, dependencies, "test-binary", name) + default: + t.Fatalf("%s has no shared binary dependency", name) + } + artifact, executable := false, false + for _, step := range job.Steps { + artifact = artifact || strings.HasPrefix(step.Uses, "actions/download-artifact@") + executable = executable || strings.Contains(step.Run, "DDTEST_TEST_BINARY=") + } + require.True(t, artifact, name) + require.True(t, executable, name) + } +} + func ciTestJobs(t *testing.T) map[string]ciTestJob { t.Helper() - data, err := os.ReadFile("../../.github/workflows/ci-full.yml") + data, err := os.ReadFile("../../.github/workflows/ci.yml") require.NoError(t, err) var workflow struct{ Jobs map[string]ciTestJob } require.NoError(t, yaml.Unmarshal(data, &workflow)) diff --git a/internal/releaseworkflow/node_fixture_test.go b/internal/releaseworkflow/node_fixture_test.go new file mode 100644 index 000000000..db63d8b1c --- /dev/null +++ b/internal/releaseworkflow/node_fixture_test.go @@ -0,0 +1,70 @@ +package releaseworkflow + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +func TestNodeFixtureCacheUsesTheWholeLockfileAndExactRuntime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture preparation runs in Ubuntu bash") + } + if _, err := exec.LookPath("sha256sum"); err != nil { + t.Skip("fixture preparation requires sha256sum") + } + data, err := os.ReadFile("../../.github/actions/node-test-fixture/action.yml") + require.NoError(t, err) + var action struct { + Runs struct { + Steps []struct { + ID, Run, If, Uses string + With map[string]string + } + } + } + require.NoError(t, yaml.Unmarshal(data, &action)) + steps := action.Runs.Steps + require.Len(t, steps, 4) + require.Contains(t, steps[1].With["key"], "runner.os") + require.Contains(t, steps[1].With["key"], "runner.arch") + require.Empty(t, steps[1].With["restore-keys"]) + require.Equal(t, "steps.modules.outputs.cache-hit != 'true'", steps[2].If) + require.Contains(t, steps[2].Run, "npm ci") + require.Equal(t, steps[1].With["key"], steps[3].With["key"]) + require.Equal(t, steps[1].With["path"], steps[3].With["path"]) + require.Equal(t, steps[2].If, steps[3].If) + tools, fixture := t.TempDir(), t.TempDir() + lock := filepath.Join(t.TempDir(), "lock.json") + require.NoError(t, os.WriteFile(lock, []byte(`{"dependencies":{"nested":"1.0.0"}}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(tools, "node"), []byte("#!/bin/sh\necho \"$FIXTURE_NODE\"\n"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(tools, "npm"), []byte("#!/bin/sh\nif [ \"$1\" = install ]; then cp \"$FIXTURE_LOCK\" package-lock.json; else echo \"$FIXTURE_NPM\"; fi\n"), 0755)) + dependencies := "example@1.0.0" + key := func(node, npm string) string { + t.Helper() + output := filepath.Join(t.TempDir(), "output") + cmd := exec.CommandContext(t.Context(), "bash", "-eu", "-c", steps[0].Run) + cmd.Env = append(os.Environ(), "PATH="+tools+string(os.PathListSeparator)+os.Getenv("PATH"), + "FIXTURE_DIRECTORY="+fixture, "FIXTURE_DEPENDENCIES="+dependencies, "FIXTURE_LOCK="+lock, + "FIXTURE_NODE="+node, "FIXTURE_NPM="+npm, "GITHUB_OUTPUT="+output) + result, err := cmd.CombinedOutput() + require.NoError(t, err, string(result)) + data, err := os.ReadFile(output) + require.NoError(t, err) + return string(data) + } + original := key("24.0.0", "11.0.0") + require.Equal(t, original, key("24.0.0", "11.0.0")) + require.NotEqual(t, original, key("24.0.1", "11.0.0")) + require.NotEqual(t, original, key("24.0.0", "11.0.1")) + require.NoError(t, os.WriteFile(lock, []byte(`{"dependencies":{"nested":"1.0.1"}}`), 0600)) + dependencies = "" + require.Equal(t, original, key("24.0.0", "11.0.0"), "a supplied lockfile must not be resolved again") + dependencies = "example@1.0.0" + require.NotEqual(t, original, key("24.0.0", "11.0.0")) +} diff --git a/internal/testdrive/apply_cucumber_integration_test.go b/internal/testdrive/apply_cucumber_integration_test.go index b8797f3a9..c8f918fca 100644 --- a/internal/testdrive/apply_cucumber_integration_test.go +++ b/internal/testdrive/apply_cucumber_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicCucumberApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicCucumberApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_mocha_integration_test.go b/internal/testdrive/apply_mocha_integration_test.go index 236e09e75..221657ec9 100644 --- a/internal/testdrive/apply_mocha_integration_test.go +++ b/internal/testdrive/apply_mocha_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicMochaApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicMochaApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_webdriverio_integration_test.go b/internal/testdrive/apply_webdriverio_integration_test.go index 2fc0e7ba2..7e16327f0 100644 --- a/internal/testdrive/apply_webdriverio_integration_test.go +++ b/internal/testdrive/apply_webdriverio_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -34,7 +35,8 @@ func TestPublicWebdriverIOApply(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -45,8 +47,6 @@ func TestPublicWebdriverIOApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_workflow_integration_test.go b/internal/testdrive/apply_workflow_integration_test.go index 918c43b45..586e4318d 100644 --- a/internal/testdrive/apply_workflow_integration_test.go +++ b/internal/testdrive/apply_workflow_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicOnboardApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicOnboardApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/configurations_integration_test.go b/internal/testdrive/configurations_integration_test.go index 8eee4a411..0e91ab393 100644 --- a/internal/testdrive/configurations_integration_test.go +++ b/internal/testdrive/configurations_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -40,9 +41,8 @@ func TestPublicJestConfigurations(t *testing.T) { cmd.Env = environment return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/coordinator_workflow_integration_test.go b/internal/testdrive/coordinator_workflow_integration_test.go index 727fb436b..f8bff6201 100644 --- a/internal/testdrive/coordinator_workflow_integration_test.go +++ b/internal/testdrive/coordinator_workflow_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,11 +29,8 @@ func TestPublicCoordinatorJestFactsAndFreshReport(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - build := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") - build.Dir = "../.." - data, err := build.CombinedOutput() - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte root := workspace.CanonicalRoot(t.TempDir()) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) write := func(name, body string) { diff --git a/internal/testdrive/cucumber_workflow_integration_test.go b/internal/testdrive/cucumber_workflow_integration_test.go index d8cff6df5..0daa00acf 100644 --- a/internal/testdrive/cucumber_workflow_integration_test.go +++ b/internal/testdrive/cucumber_workflow_integration_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -32,7 +33,8 @@ func TestPublicCucumberTestdrive(t *testing.T) { require.NoError(t, json.Unmarshal(manifest, &installed)) ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte var hookFile string newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) @@ -48,8 +50,6 @@ func TestPublicCucumberTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range cucumberWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { if _, err := os.Stat(filepath.Join(modules, "tsx")); mode == "source loader" && err != nil { diff --git a/internal/testdrive/cypress_workflow_integration_test.go b/internal/testdrive/cypress_workflow_integration_test.go index 13c3a3667..df414fa83 100644 --- a/internal/testdrive/cypress_workflow_integration_test.go +++ b/internal/testdrive/cypress_workflow_integration_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -27,7 +28,8 @@ func TestPublicCypressTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 28*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -41,8 +43,6 @@ func TestPublicCypressTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range cypressWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/forwarded_configuration_integration_test.go b/internal/testdrive/forwarded_configuration_integration_test.go index 71d3d84e4..f6e862108 100644 --- a/internal/testdrive/forwarded_configuration_integration_test.go +++ b/internal/testdrive/forwarded_configuration_integration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -59,9 +60,8 @@ func TestPublicJestForwardedConfiguration(t *testing.T) { cmd.Env = append(cmd.Env, "NODE_OPTIONS=") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, flags := range [][]string{{}, {"--command", "npm test"}} { diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 7b978b8e9..81dbf119f 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -26,8 +27,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) fixtures := []struct { name, manifest, command string files map[string]string diff --git a/internal/testdrive/java_validation_integration_test.go b/internal/testdrive/java_validation_integration_test.go index 461dd70e9..d0ae330c1 100644 --- a/internal/testdrive/java_validation_integration_test.go +++ b/internal/testdrive/java_validation_integration_test.go @@ -19,6 +19,7 @@ import ( "time" shard "github.com/DataDog/ddtest/internal/testdrive" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -32,8 +33,7 @@ func TestJavaValidationIntegration(t *testing.T) { tracer := cmp.Or(os.Getenv("DDTEST_JAVA_TRACER_VERSION"), "1.67.1") ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) type javaReport struct { Success bool diff --git a/internal/testdrive/jest_workflow_integration_test.go b/internal/testdrive/jest_workflow_integration_test.go index 2fb1efc05..34e549671 100644 --- a/internal/testdrive/jest_workflow_integration_test.go +++ b/internal/testdrive/jest_workflow_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -25,11 +26,8 @@ func TestPublicJestTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 12*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - build := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") - build.Dir = "../.." - output, err := build.CombinedOutput() - require.NoError(t, err, string(output)) + binary := testutil.DDTest(t) + var output []byte _, tsErr := os.Stat(filepath.Join(modules, "ts-jest")) for _, mode := range testdrive.JestWorkflowScenarios(t) { if strings.HasPrefix(mode, "TypeScript") && tsErr != nil { diff --git a/internal/testdrive/languages_integration_test.go b/internal/testdrive/languages_integration_test.go index fc9422ff6..3f0711195 100644 --- a/internal/testdrive/languages_integration_test.go +++ b/internal/testdrive/languages_integration_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,8 +29,7 @@ func TestPublicPythonRubyTestdrives(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 18*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) for _, fixture := range []struct { name, command string files map[string]string diff --git a/internal/testdrive/loader_configuration_integration_test.go b/internal/testdrive/loader_configuration_integration_test.go index 011bc474e..52b7b3f60 100644 --- a/internal/testdrive/loader_configuration_integration_test.go +++ b/internal/testdrive/loader_configuration_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -65,9 +66,8 @@ Module._resolveFilename=function(request,...args){return entries[request]||origi cmd.Env = append(cmd.Env, "NODE_OPTIONS=") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, flags := range [][]string{{"--check-only"}, {}} { diff --git a/internal/testdrive/mocha_workflow_integration_test.go b/internal/testdrive/mocha_workflow_integration_test.go index 867bf0814..4f0a97b98 100644 --- a/internal/testdrive/mocha_workflow_integration_test.go +++ b/internal/testdrive/mocha_workflow_integration_test.go @@ -3,6 +3,7 @@ package testdrive import ( "context" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" "os" "os/exec" @@ -21,7 +22,8 @@ func TestPublicMochaTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 8*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -32,8 +34,6 @@ func TestPublicMochaTestdrive(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range mochaWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/package_configuration_integration_test.go b/internal/testdrive/package_configuration_integration_test.go index 0e4aec123..cac59f437 100644 --- a/internal/testdrive/package_configuration_integration_test.go +++ b/internal/testdrive/package_configuration_integration_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/kballard/go-shellquote" "github.com/stretchr/testify/require" "os" @@ -82,9 +83,7 @@ func TestPublicJestPackageConfiguration(t *testing.T) { cmd.Env = append(cmd.Env, "NODE_OPTIONS=--conditions=development") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err = command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, args := range [][]string{{"--all", "--check-only"}, {"--all"}} { @@ -256,9 +255,7 @@ func TestPublicJestTwoPackageConfiguration(t *testing.T) { } data, err := command(root, "git", "init", "-q") require.NoError(t, err, string(data)) - binary := filepath.Join(t.TempDir(), "ddtest") - data, err = command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) for _, args := range [][]string{{"--all", "--check-only"}, {"--all"}} { data, err = command(root, binary, append([]string{"testdrive", "--framework", "jest", "--yes"}, args...)...) require.NoError(t, err, string(data)) diff --git a/internal/testdrive/playwright_workflow_integration_test.go b/internal/testdrive/playwright_workflow_integration_test.go index df0b3f738..4e165e5b0 100644 --- a/internal/testdrive/playwright_workflow_integration_test.go +++ b/internal/testdrive/playwright_workflow_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,7 +29,8 @@ func TestPublicPlaywrightTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 15*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -39,8 +41,6 @@ func TestPublicPlaywrightTestdrive(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range playwrightWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/preflight_review_test.go b/internal/testdrive/preflight_review_test.go index 6557b1ae3..47f681e99 100644 --- a/internal/testdrive/preflight_review_test.go +++ b/internal/testdrive/preflight_review_test.go @@ -152,7 +152,7 @@ func TestRecommendationsUseRegistryTagsAndFlagUnknownMajors(t *testing.T) { } func TestJestSupportRulesCoverCICompatibilityMatrix(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/ci-full.yml") + data, err := os.ReadFile("../../.github/workflows/ci.yml") require.NoError(t, err) var workflow struct { Jobs map[string]struct { diff --git a/internal/testdrive/python_feedback_test.go b/internal/testdrive/python_feedback_test.go index e05dd9d10..22dfbe9bb 100644 --- a/internal/testdrive/python_feedback_test.go +++ b/internal/testdrive/python_feedback_test.go @@ -80,6 +80,26 @@ func TestPythonValidationProbesRespectTimeout(t *testing.T) { } } +func TestPythonPrerequisiteProbeDoesNotStartDatadogReporting(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX executable fixture") + } + directory := t.TempDir() + result := filepath.Join(directory, "probe-environment") + python := filepath.Join(directory, "python") + require.NoError(t, os.WriteFile(python, []byte("#!/bin/sh\nprintf '%s %s %s' \"$DD_INSTRUMENTATION_TELEMETRY_ENABLED\" \"$DD_TRACE_ENABLED\" \"$DD_CIVISIBILITY_ENABLED\" > \"$PROBE_ENVIRONMENT\"\n"), 0755)) + t.Setenv("PROBE_ENVIRONMENT", result) + t.Setenv("DDTEST_PYTEST_PYTHON", python) + for _, key := range []string{"DD_INSTRUMENTATION_TELEMETRY_ENABLED", "DD_TRACE_ENABLED", "DD_CIVISIBILITY_ENABLED"} { + t.Setenv(key, "true") + } + require.Equal(t, python, feedbackPython(t)) + data, err := os.ReadFile(result) + require.NoError(t, err) + require.Equal(t, "false false false", string(data)) + require.Equal(t, "true", os.Getenv("DD_CIVISIBILITY_ENABLED")) +} + func feedbackPython(t *testing.T) string { t.Helper() python := os.Getenv("DDTEST_PYTEST_PYTHON") @@ -89,7 +109,11 @@ func feedbackPython(t *testing.T) string { if python == "" { t.Skip("Python required") } - if output, err := exec.CommandContext(t.Context(), python, "-c", `import pytest, ddtrace; from packaging.version import Version; assert Version(ddtrace.__version__) >= Version("4.11.0") and Version(pytest.__version__) >= Version("7.2")`).CombinedOutput(); err != nil { + probe := exec.CommandContext(t.Context(), python, "-c", `import pytest, ddtrace; from packaging.version import Version; assert Version(ddtrace.__version__) >= Version("4.11.0") and Version(pytest.__version__) >= Version("7.2")`) + // This process only checks installed versions. The real validation processes + // still exercise telemetry and all feature controls against the local intake. + probe.Env = append(os.Environ(), "DD_INSTRUMENTATION_TELEMETRY_ENABLED=false", "DD_TRACE_ENABLED=false", "DD_CIVISIBILITY_ENABLED=false") + if output, err := probe.CombinedOutput(); err != nil { if os.Getenv("DDTEST_REQUIRE_PROJECT_RUNTIMES") == "1" || os.Getenv("DDTEST_PYTEST_PYTHON") != "" { t.Fatalf("pytest and ddtrace required: %s: %v", output, err) } diff --git a/internal/testdrive/ruby_fork_integration_test.go b/internal/testdrive/ruby_fork_integration_test.go index 017c91279..3b28c8144 100644 --- a/internal/testdrive/ruby_fork_integration_test.go +++ b/internal/testdrive/ruby_fork_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -44,8 +45,7 @@ func TestRubyMinitestForkValidationIntegration(t *testing.T) { integrationFile(t, root, ".github/workflows/test.yml", "jobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: ruby/setup-ruby@v1\n with: {ruby-version: '"+installed.Ruby+"'}\n - uses: datadog/test-visibility-github-action@v3\n with: {languages: ruby, api_key: '${{ secrets.DD_API_KEY }}', ruby-tracer-version: '"+installed.Gems["datadog-ci"]+"'}\n - run: bundle exec rake test\n env:\n RUBYOPT: '${{ env.RUBYOPT }} -r./ci/datadog_test_optimization.rb'\n") integrationCommand(t, ctx, root, nil, "bundle", "lock", "--local") integrationCommand(t, ctx, root, nil, "git", "add", ".") - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) env := []string{"DD_GIT_REPOSITORY_URL=https://example.com/ddtest-isolation-fixture.git", "DD_GIT_COMMIT_SHA=" + strings.Repeat("1", 40), "RUBYOPT=-W0"} protocol := rubyProtocolCommand(t, ctx, root, env, binary, "testdrive", "--framework", "minitest", "--all", "--agent", "--yes", "--tracer-version", installed.Gems["datadog-ci"]) var state struct{ Status string } diff --git a/internal/testdrive/ruby_nested_integration_test.go b/internal/testdrive/ruby_nested_integration_test.go index 957b5e0bb..92cb5833b 100644 --- a/internal/testdrive/ruby_nested_integration_test.go +++ b/internal/testdrive/ruby_nested_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -23,8 +24,7 @@ func TestRubyNestedWorkingDirectoryIntegration(t *testing.T) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby:RUBY_VERSION,minitest:Gem::Specification.find_by_name("minitest").version.to_s,rake:Gem::Specification.find_by_name("rake").version.to_s,tracer:Gem::Specification.find_by_name("datadog-ci").version.to_s)`) var gems struct{ Ruby, Minitest, Rake, Tracer string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() retainIntegrationReports(t, root, "minitest-nested-review") integrationCommand(t, ctx, root, nil, "git", "init", "-q") diff --git a/internal/testdrive/ruby_receipt_integration_test.go b/internal/testdrive/ruby_receipt_integration_test.go index a21319220..547a9286a 100644 --- a/internal/testdrive/ruby_receipt_integration_test.go +++ b/internal/testdrive/ruby_receipt_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,8 +23,7 @@ func TestRubyReceiptReviewIntegration(t *testing.T) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby:RUBY_VERSION,minitest:Gem::Specification.find_by_name("minitest").version.to_s,rake:Gem::Specification.find_by_name("rake").version.to_s,tracer:Gem::Specification.find_by_name("datadog-ci").version.to_s)`) var gems struct{ Ruby, Minitest, Rake, Tracer string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() retainIntegrationReports(t, root, "minitest-receipts") integrationCommand(t, ctx, root, nil, "git", "init", "-q") diff --git a/internal/testdrive/ruby_validation_integration_test.go b/internal/testdrive/ruby_validation_integration_test.go index 4d182f942..c797d3a3a 100644 --- a/internal/testdrive/ruby_validation_integration_test.go +++ b/internal/testdrive/ruby_validation_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -29,8 +30,7 @@ func runRubyValidationIntegration(t *testing.T, frameworks ...string) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby: RUBY_VERSION, rspec: Gem::Specification.find_by_name("rspec-core").version.to_s, minitest: Gem::Specification.find_by_name("minitest").version.to_s, tracer: Gem::Specification.find_by_name("datadog-ci").version.to_s, rake: Gem::Specification.find_by_name("rake").version.to_s)`) var gems struct{ Ruby, Rspec, Minitest, Tracer, Rake string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) for _, framework := range frameworks { t.Run(framework, func(t *testing.T) { root := filepath.Join(t.TempDir(), "project space") diff --git a/internal/testdrive/vitest_review_test.go b/internal/testdrive/vitest_review_test.go index 7b17c1754..24033b506 100644 --- a/internal/testdrive/vitest_review_test.go +++ b/internal/testdrive/vitest_review_test.go @@ -208,7 +208,7 @@ func TestVitestRejectedCommandPublishesCurrentFailure(t *testing.T) { } func TestVitestCIMatrixUsesCompatibleTracerFloors(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/ci-full.yml") + data, err := os.ReadFile("../../.github/workflows/ci.yml") require.NoError(t, err) var ci struct { Jobs map[string]struct { diff --git a/internal/testdrive/vitest_runner_integration_test.go b/internal/testdrive/vitest_runner_integration_test.go index fffcf04c8..905331edb 100644 --- a/internal/testdrive/vitest_runner_integration_test.go +++ b/internal/testdrive/vitest_runner_integration_test.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/DataDog/ddtest/internal/testutil" "os" "os/exec" "path/filepath" @@ -26,8 +27,7 @@ func TestPublicVitestRunnerTestdrive(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest-under-test") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() integrationCommand(t, ctx, root, nil, "git", "init", "-q") vitest, tracer := os.Getenv("DDTEST_VITEST_VERSION"), os.Getenv("DDTEST_VITEST_TRACER_VERSION") diff --git a/internal/testdrive/webdriverio_workflow_integration_test.go b/internal/testdrive/webdriverio_workflow_integration_test.go index edf987897..93a514a0a 100644 --- a/internal/testdrive/webdriverio_workflow_integration_test.go +++ b/internal/testdrive/webdriverio_workflow_integration_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -44,7 +45,8 @@ func TestPublicWebdriverIOTestdrive(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 24*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -58,8 +60,6 @@ func TestPublicWebdriverIOTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) // Process sockets live under /tmp, not TMPDIR; collect the folders that exist // before a run so later checks only judge the ones the run created. processFolders := func() map[string]bool { diff --git a/internal/testutil/binary.go b/internal/testutil/binary.go new file mode 100644 index 000000000..ed3161667 --- /dev/null +++ b/internal/testutil/binary.go @@ -0,0 +1,65 @@ +// Package testutil shares expensive fixtures between the CLI integration tests. +package testutil + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" +) + +type binaryCache struct { + once sync.Once + path string + err error +} + +var compiled = new(binaryCache) + +// Capture the package directory before customer fixtures use t.Chdir. +var repositoryRoot, repositoryRootError = filepath.Abs("../..") + +// DDTest uses the current CI run's binary, or builds it once per test process. +// TestMain's isolated temporary directory owns the locally built executable. +func DDTest(t testing.TB) string { + t.Helper() + if path := os.Getenv("DDTEST_TEST_BINARY"); path != "" { + info, err := os.Stat(path) + if !filepath.IsAbs(path) || err != nil || !info.Mode().IsRegular() { + t.Fatalf("DDTEST_TEST_BINARY must name an existing absolute executable: %q", path) + } + return path + } + compiled.once.Do(func() { compiled.path, compiled.err = buildDDTest(t.Context()) }) + if compiled.err != nil { + t.Fatal(compiled.err) + } + return compiled.path +} + +func buildDDTest(ctx context.Context) (string, error) { + if repositoryRootError != nil { + return "", repositoryRootError + } + directory, err := os.MkdirTemp("", "ddtest-binary-") + if err != nil { + return "", err + } + name := "ddtest" + if runtime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(directory, name) + command := exec.CommandContext(ctx, "go", "build", "-o", path, ".") + command.Dir = repositoryRoot + command.Env = append(os.Environ(), "CGO_ENABLED=0") + if output, err := command.CombinedOutput(); err != nil { + _ = os.RemoveAll(directory) + return "", fmt.Errorf("build integration CLI: %w\n%s", err, output) + } + return path, nil +} diff --git a/internal/testutil/binary_test.go b/internal/testutil/binary_test.go new file mode 100644 index 000000000..9733fece4 --- /dev/null +++ b/internal/testutil/binary_test.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDDTestBuildIsSharedAcrossWorkingDirectories(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX fake Go executable") + } + previous := compiled + compiled = new(binaryCache) + t.Cleanup(func() { compiled = previous }) + t.Setenv("DDTEST_TEST_BINARY", "") + directory := t.TempDir() + t.Setenv("TMPDIR", directory) + log := filepath.Join(directory, "builds") + t.Setenv("BUILD_LOG", log) + goCommand := "#!/bin/sh\nprintf '%s\\n' \"$PWD\" >> \"$BUILD_LOG\"\nprintf '#!/bin/sh\\nexit 0\\n' > \"$3\"\nchmod +x \"$3\"\n" + require.NoError(t, os.WriteFile(filepath.Join(directory, "go"), []byte(goCommand), 0755)) + t.Setenv("PATH", directory+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Chdir(t.TempDir()) + first := DDTest(t) + t.Chdir(t.TempDir()) + require.Equal(t, first, DDTest(t)) + data, err := os.ReadFile(log) + require.NoError(t, err) + require.Len(t, strings.Split(strings.TrimSpace(string(data)), "\n"), 1) + require.FileExists(t, filepath.Join(strings.TrimSpace(string(data)), "go.mod")) + info, err := os.Stat(first) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) +} + +func TestDDTestUsesTheProvidedCIExecutable(t *testing.T) { + path := filepath.Join(t.TempDir(), "ddtest") + require.NoError(t, os.WriteFile(path, []byte("fixture"), 0755)) + t.Setenv("DDTEST_TEST_BINARY", path) + t.Setenv("PATH", t.TempDir()) // A provided executable never needs Go installed. + t.Chdir(t.TempDir()) + require.Equal(t, path, DDTest(t)) +} diff --git a/scripts/ci_smoke.py b/scripts/ci_smoke.py deleted file mode 100644 index 89e93c45c..000000000 --- a/scripts/ci_smoke.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Run a pinned PR integration smoke and require its named cases to pass.""" - -import argparse -import json -import os -from pathlib import Path -import subprocess -import tempfile - -from ci_test_plan import CONFIG - - -def require_cases(passed, required): - missing = set(required) - passed - if missing: - raise ValueError(f"Smoke cases did not pass (missing, skipped or filtered out): {sorted(missing)}") - - -def run_tests(command, env, required): - passed = set() - with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as process: - for line in process.stdout: - try: - event = json.loads(line) - except ValueError: - print(line, end="", flush=True) - continue - if event.get("Action") == "pass" and event.get("Test"): - passed.add(event["Test"]) - if event.get("Output"): - print(event["Output"], end="", flush=True) - if process.wait(): - raise subprocess.CalledProcessError(process.returncode, command) - require_cases(passed, required) - - -def run_smoke(entry): - env = {**os.environ, **entry.get("env", {}), "DDTEST_TEST_SHARD": "1/1"} - with tempfile.TemporaryDirectory(prefix="ddtest-smoke-", dir=os.environ.get("RUNNER_TEMP")) as directory: - if entry.get("dependencies"): - subprocess.run(["npm", "install", "--prefix", directory, "--no-audit", "--no-fund"] - + entry["dependencies"], check=True) - for variable in entry["modules"]: - env[variable] = str(Path(directory) / "node_modules") - for command in entry.get("setup", []): - subprocess.run(command, check=True, env=env) - command = (["xvfb-run", "-a"] if entry.get("xvfb") else []) + [ - "go", "test", "-json", "-count=1", f"-timeout={entry['timeout_minutes']}m", "-run", entry["tests"], - ] + entry.get("packages", ["./internal/testdrive"]) - run_tests(command, env, entry["required"]) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--id", required=True) - args = parser.parse_args() - config = json.loads(CONFIG.read_text()) - entries = {entry["id"]: entry for suite in config["suites"].values() for entry in suite["smoke"]} - if args.id not in entries: - parser.error(f"unknown smoke: {args.id}") - run_smoke(entries[args.id]) - - -if __name__ == "__main__": - main() diff --git a/scripts/ci_test_plan.json b/scripts/ci_test_plan.json deleted file mode 100644 index b05b5461d..000000000 --- a/scripts/ci_test_plan.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "non_product": ["skills/ddtest-onboarding-eval/**"], - "ignored": [ - "docs/**", - "README.md", - "AGENTS.md", - "ddto/docs/**", - "ddto/AGENTS.md", - "ddto/README.md" - ], - "shared": [ - "internal/testdrive/cucumber_coverage.go", - "internal/testdrive/cypress_coverage.go", - "internal/testdrive/jest*.go", - "internal/testdrive/mocha.go", - "internal/onboard/jest_command.go", - "internal/onboard/python_handwritten.go" - ], - "runtime_test_inputs": [ - "**/*integration_test.go", - "internal/compatibility/**", - "internal/framework/webdriverio_test.go", - "internal/platform/javascript_test.go", - "internal/testdrive/cucumber_support_test.go", - "internal/testdrive/jest_preflight_test.go", - "internal/testdrive/mocha_validation_test.go", - "internal/testdrive/vitest_review_test.go", - "internal/testdrive/vitest_test.go", - "internal/testdrive/webdriverio_followup_test.go", - "internal/testdrive/webdriverio_review_test.go", - "internal/testdrive/webdriverio_test.go" - ], - "suites": { - "python": { - "jobs": [ - "pytest-compatibility", - "testdrive-language-workflow" - ], - "inputs": [ - "internal/compatibility/pytest*", - "internal/framework/pytest*", - "internal/platform/python*", - "internal/platform/scripts/python_*", - "internal/onboard/python*", - "internal/onboard/instructions/pytest.md", - "internal/testdrive/python*", - "internal/testdrive/pytest*", - "internal/testdrive/scripts/pytest_*" - ], - "smoke": [ - { - "id": "python", - "runtime": "python", - "timeout_minutes": 20, - "tests": "^TestPublicPythonRubyTestdrives$/^pytest$", - "required": [ - "TestPublicPythonRubyTestdrives/pytest" - ], - "env": { - "DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST": "1" - } - } - ] - }, - "ruby": { - "jobs": [ - "rspec-compatibility", - "minitest-compatibility", - "ruby-testdrive" - ], - "inputs": [ - "internal/compatibility/rspec*", - "internal/compatibility/minitest*", - "internal/framework/rspec*", - "internal/framework/minitest*", - "internal/platform/ruby*", - "internal/platform/scripts/ruby_*", - "internal/onboard/ruby*", - "internal/onboard/instructions/ruby*", - "internal/onboard/python_ruby_parity_test.go", - "internal/testdrive/ruby*", - "internal/testdrive/scripts/ruby_*", - "internal/testdrive/python_ruby_parity_test.go" - ], - "smoke": [ - { - "id": "ruby", - "runtime": "ruby", - "timeout_minutes": 20, - "tests": "^TestRubyValidationIntegration$/^(rspec|minitest)$", - "required": [ - "TestRubyValidationIntegration/rspec", - "TestRubyValidationIntegration/minitest" - ], - "env": { - "DDTEST_RUN_RUBY_INTEGRATION_TEST": "1" - }, - "setup": [ - [ - "gem", - "install", - "datadog-ci", - "--version", - "1.40.0", - "--no-document" - ], - [ - "gem", - "install", - "rspec", - "--version", - "3.13.2", - "--no-document" - ], - [ - "gem", - "install", - "minitest", - "--version", - "6.0.6", - "--no-document" - ], - [ - "gem", - "install", - "rake", - "--version", - "13.4.2", - "--no-document" - ] - ] - } - ] - }, - "java": { - "jobs": [ - "java-testdrive" - ], - "inputs": [ - "internal/framework/jvm*", - "internal/platform/java[._]*", - "internal/gradle/**", - "internal/maven/**", - "internal/onboard/java[._]*", - "internal/onboard/instructions/java*", - "internal/testdrive/java[._]*" - ], - "smoke": [ - { - "id": "java-maven", - "runtime": "java", - "timeout_minutes": 20, - "tests": "^TestJavaValidationIntegration$/^junit4$", - "required": [ - "TestJavaValidationIntegration/junit4" - ], - "env": { - "DDTEST_RUN_JAVA_INTEGRATION_TEST": "1", - "DDTEST_JAVA_TRACER_VERSION": "1.67.1", - "MAVEN_OPTS": "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" - } - }, - { - "id": "java-gradle", - "runtime": "gradle", - "timeout_minutes": 20, - "tests": "^TestJavaValidationIntegration$/^gradle_junit4$", - "required": [ - "TestJavaValidationIntegration/gradle_junit4" - ], - "env": { - "DDTEST_RUN_JAVA_INTEGRATION_TEST": "1", - "DDTEST_JAVA_TRACER_VERSION": "1.67.1", - "MAVEN_OPTS": "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" - } - } - ] - }, - "javascript": { - "jobs": [ - "javascript-platform-compatibility" - ], - "inputs": [ - "internal/platform/javascript*", - "internal/platform/node_options*", - "internal/platform/scripts/javascript*", - "internal/framework/javascript*", - "internal/onboard/javascript*", - "internal/onboard/node_*", - "internal/testdrive/javascript*" - ], - "smoke": [ - { - "id": "javascript", - "runtime": "node", - "timeout_minutes": 10, - "packages": [ - "./internal/compatibility" - ], - "tests": "^TestJavaScript(Platform|ActionPreload)Integration$", - "required": [ - "TestJavaScriptPlatformIntegration", - "TestJavaScriptActionPreloadIntegration" - ], - "dependencies": [ - "dd-trace@6.15.0" - ], - "modules": [ - "DDTEST_DD_TRACE_NODE_MODULES" - ] - } - ] - }, - "jest": { - "jobs": [ - "jest-compatibility", - "jest-testdrive-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/jest*", - "internal/framework/jest*", - "internal/onboard/jest*", - "internal/testdrive/jest*" - ], - "smoke": [ - { - "id": "jest", - "runtime": "node", - "timeout_minutes": 12, - "tests": "^TestPublicJestTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicJestTestdrive/passing", - "TestPublicJestTestdrive/existing_failure" - ], - "dependencies": [ - "jest@30.5.1", - "dd-trace@6.15.0", - "typescript@5" - ], - "modules": [ - "DDTEST_JEST_NODE_MODULES" - ] - } - ] - }, - "vitest": { - "jobs": [ - "vitest-compatibility", - "testdrive-vitest-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/vitest*", - "internal/framework/vitest*", - "internal/framework/scripts/vitest*", - "internal/onboard/vitest*", - "internal/onboard/instructions/vitest.md", - "internal/testdrive/vitest*", - "internal/testdrive/scripts/vitest*" - ], - "smoke": [ - { - "id": "vitest", - "runtime": "node", - "timeout_minutes": 15, - "tests": "^TestPublicVitestRunnerTestdrive$/^mixed_CI_paths$", - "required": [ - "TestPublicVitestRunnerTestdrive/mixed_CI_paths" - ], - "dependencies": [ - "vitest@4.1.11", - "@vitest/coverage-v8@4.1.11", - "dd-trace@6.21.0" - ], - "modules": [ - "DDTEST_VITEST_NODE_MODULES" - ], - "env": { - "DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST": "1" - } - } - ] - }, - "mocha": { - "jobs": [ - "mocha-compatibility", - "testdrive-mocha-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/mocha*", - "internal/framework/mocha*", - "internal/framework/scripts/mocha*", - "internal/onboard/mocha*", - "internal/onboard/instructions/mocha.md", - "internal/testdrive/mocha*", - "internal/testdrive/scripts/mocha*" - ], - "smoke": [ - { - "id": "mocha", - "runtime": "node", - "timeout_minutes": 10, - "tests": "^TestPublicMochaTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicMochaTestdrive/passing", - "TestPublicMochaTestdrive/existing_failure" - ], - "dependencies": [ - "mocha@11.7.6", - "nyc@17.1.0", - "c8@12.0.0", - "dd-trace@6.21.0" - ], - "modules": [ - "DDTEST_MOCHA_NODE_MODULES" - ] - } - ] - }, - "cypress": { - "jobs": [ - "cypress-compatibility", - "testdrive-cypress-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/cypress*", - "internal/framework/cypress*", - "internal/framework/scripts/cypress*", - "internal/onboard/cypress*", - "internal/onboard/instructions/cypress.md", - "internal/testdrive/cypress*", - "internal/testdrive/scripts/cypress*" - ], - "smoke": [ - { - "id": "cypress", - "runtime": "node", - "timeout_minutes": 12, - "tests": "^TestPublicCypressTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicCypressTestdrive/passing", - "TestPublicCypressTestdrive/existing_failure" - ], - "dependencies": [ - "cypress@15.1.0", - "dd-trace@6.20.1", - "typescript@5.9.3" - ], - "modules": [ - "DDTEST_CYPRESS_NODE_MODULES" - ], - "xvfb": true - } - ] - }, - "playwright": { - "jobs": [ - "playwright-compatibility", - "playwright-latest", - "testdrive-playwright-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/playwright*", - "internal/framework/playwright*", - "internal/framework/scripts/playwright*", - "internal/framework/cypress.go", - "internal/onboard/playwright*", - "internal/onboard/instructions/playwright.md", - "internal/testdrive/playwright*", - "internal/testdrive/scripts/playwright*" - ], - "smoke": [ - { - "id": "playwright", - "runtime": "playwright", - "timeout_minutes": 12, - "tests": "^TestPublicPlaywrightTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicPlaywrightTestdrive/passing", - "TestPublicPlaywrightTestdrive/existing_failure" - ], - "dependencies": [ - "@playwright/test@1.55.1", - "dd-trace@6.20.1" - ], - "modules": [ - "DDTEST_PLAYWRIGHT_NODE_MODULES" - ] - } - ] - }, - "cucumber": { - "jobs": [ - "cucumber-compatibility", - "cucumber-matrix", - "testdrive-cucumber-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/compatibility/cucumber*", - "internal/framework/cucumber*", - "internal/onboard/cucumber*", - "internal/onboard/instructions/cucumber.md", - "internal/testdrive/cucumber*", - "internal/testdrive/scripts/cucumber*" - ], - "smoke": [ - { - "id": "cucumber", - "runtime": "node", - "timeout_minutes": 12, - "tests": "^TestPublicCucumberTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicCucumberTestdrive/passing", - "TestPublicCucumberTestdrive/existing_failure" - ], - "dependencies": [ - "@cucumber/cucumber@13.3.0", - "nyc@17.1.0", - "c8@12.0.0", - "dd-trace@6.20.1", - "@istanbuljs/esm-loader-hook@0.3.0", - "tsx@4.23.15" - ], - "modules": [ - "DDTEST_CUCUMBER_NODE_MODULES" - ] - } - ] - }, - "webdriverio": { - "jobs": [ - "testdrive-webdriverio-workflow" - ], - "depends_on": [ - "javascript" - ], - "inputs": [ - "internal/framework/webdriverio*", - "internal/framework/scripts/webdriverio*", - "internal/platform/webdriverio*", - "internal/onboard/webdriverio*", - "internal/onboard/instructions/webdriverio.md", - "internal/testdrive/webdriverio*", - "internal/testdrive/mocha_validation_test.go" - ], - "smoke": [ - { - "id": "webdriverio", - "runtime": "node", - "timeout_minutes": 12, - "tests": "^TestPublicWebdriverIOTestdrive$/^(passing|existing_failure)$", - "required": [ - "TestPublicWebdriverIOTestdrive/passing", - "TestPublicWebdriverIOTestdrive/existing_failure" - ], - "dependencies": [ - "@wdio/cli@9.20.0", - "@wdio/local-runner@9.20.0", - "@wdio/mocha-framework@9.20.0", - "@wdio/jasmine-framework@9.20.0", - "@wdio/spec-reporter@9.20.0", - "dd-trace@6.20.1" - ], - "modules": [ - "DDTEST_WEBDRIVERIO_NODE_MODULES" - ], - "env": { - "DDTEST_CHROME_BINARY": "/usr/bin/google-chrome" - }, - "xvfb": true - } - ] - }, - "onboarding": { - "jobs": [ - "onboard-apply-workflow" - ], - "depends_on": [ - "javascript", - "vitest", - "mocha", - "cucumber", - "webdriverio" - ], - "inputs": [ - "internal/testdrive/apply*" - ], - "additional_inputs": [ - "internal/onboard/**" - ], - "smoke": [ - { - "id": "onboarding", - "runtime": "node", - "timeout_minutes": 8, - "tests": "^TestPublicOnboardApply$", - "required": [ - "TestPublicOnboardApply" - ], - "dependencies": [ - "vitest@4.1.11", - "dd-trace@6.20.1" - ], - "modules": [ - "DDTEST_VITEST_NODE_MODULES" - ] - } - ] - } - } -} diff --git a/scripts/ci_test_plan.py b/scripts/ci_test_plan.py deleted file mode 100644 index 9f211808c..000000000 --- a/scripts/ci_test_plan.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Select PR smoke checks and affected Go packages from the actual Git diff.""" - -import argparse -from fnmatch import fnmatchcase -import json -import os -from pathlib import Path -import subprocess - - -CONFIG = Path(__file__).with_suffix(".json") - - -def affected_suites(path, config): - if any(fnmatchcase(path, pattern) for pattern in config["ignored"]): - return set() - suites = config["suites"] - if not path.endswith("_test.go") and any(fnmatchcase(path, pattern) for pattern in config["shared"]): - return set(suites) - affected = {name for name, suite in suites.items() - if any(fnmatchcase(path, pattern) for pattern in suite["inputs"])} - if not affected: - return set(suites) - affected.update(name for name, suite in suites.items() - if any(fnmatchcase(path, pattern) for pattern in suite.get("additional_inputs", []))) - while True: - expanded = affected | {name for name, suite in suites.items() - if affected.intersection(suite.get("depends_on", []))} - if expanded == affected: - return affected - affected = expanded - - -def relevant_files(files, config): - if files is None: - return None - ignored = config["ignored"] + config.get("non_product", []) - return [path for path in files if not any(fnmatchcase(path, pattern) for pattern in ignored)] - - -def changed_files(root, base): - if not base or base == "0" * 40: - return None - try: - ancestor = subprocess.run(["git", "merge-base", base, "HEAD"], cwd=root, - check=True, capture_output=True, text=True).stdout.strip() - tracked = subprocess.run(["git", "diff", "--no-renames", "--name-only", "-z", ancestor], - cwd=root, check=True, capture_output=True).stdout - added = subprocess.run(["git", "ls-files", "--others", "--exclude-standard", "-z"], - cwd=root, check=True, capture_output=True).stdout - return sorted(set(os.fsdecode(tracked + added).split("\0")) - {""}) - except subprocess.CalledProcessError: - return None - - -def make_plan(files, config): - selected = set(config["suites"]) if files is None else set() - relevant = relevant_files(files, config) - for path in relevant or []: - # The instrumentation is a separate module. Its units and the release - # binary smoke cover changes there; they do not affect framework runners. - if path.startswith("ddto/"): - continue - if path.endswith("_test.go") and not any(fnmatchcase(path, pattern) - for pattern in config["runtime_test_inputs"]): - continue - selected.update(affected_suites(path, config)) - checks = [{"id": entry["id"], "runtime": entry["runtime"]} - for name, suite in config["suites"].items() if name in selected - for entry in suite["smoke"]] - return { - "mode": "fast", "changed_files": files, - "go": files is None or bool(relevant), - "ddto": files is None or any(path.startswith("ddto/") or path in ( - "main.go", "go.mod", "go.sum", "internal/cmd/instrument.go", - ".github/workflows/ci.yml", ".github/workflows/go-instrumentation.yml") for path in relevant), - "evaluator": files is None or any(path == ".github/workflows/ci.yml" or path.startswith("skills/ddtest-onboarding-eval/") - for path in files), - "suites": sorted(selected), - "integrations": [entry for entry in checks if entry["runtime"] != "playwright"], - "playwright": any(entry["runtime"] == "playwright" for entry in checks), - } - - -def json_objects(text): - decoder = json.JSONDecoder() - while text.strip(): - value, end = decoder.raw_decode(text.lstrip()) - yield value - text = text.lstrip()[end:] - - -def select_packages(packages, files, root): - by_dir = {Path(package["Dir"]).resolve(): package["ImportPath"] for package in packages} - all_packages = set(by_dir.values()) - if files is None: - return sorted(all_packages) - direct, production = set(), set() - for path in files: - if not path.endswith(".go") or path.startswith("ddto/"): - return sorted(all_packages) - package = by_dir.get((root / path).resolve().parent) - if package is None: - return sorted(all_packages) - direct.add(package) - if not path.endswith("_test.go"): - production.add(package) - while True: - expanded = production | {package["ImportPath"] for package in packages - if production.intersection(package.get("Imports", []) - + package.get("TestImports", []) - + package.get("XTestImports", []))} - if expanded == production: - return sorted(direct | production) - production = expanded - - -def run_unit_tests(root, files, config, race=False): - relevant = relevant_files(files, config) - if relevant == []: - print("No code inputs changed; no Go packages selected.") - return - command = ["go", "list", "-json"] + (["-race"] if race else []) + ["./..."] - result = subprocess.run(command, cwd=root, check=True, capture_output=True, text=True) - packages = select_packages(list(json_objects(result.stdout)), relevant, root) - if not packages: - raise ValueError("Go reported no packages for changed code") - print(f"Testing {len(packages)} affected Go packages", flush=True) - subprocess.run(["go", "test", "-timeout=20m"] + (["-race"] if race else []) + packages, - cwd=root, check=True) - - -def summary(plan): - suites = ", ".join(plan["suites"]) or "none (documentation only)" - return ("## PR test selection\n\n" - f"Selected integration suites: {suites}.\n\n" - f"Native Go checks: {plan['go']}; instrumentation units: {plan['ddto']}.\n\n" - "Historical versions, full scenario matrices and platform parity gate releases " - "through the Full compatibility workflow.\n") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=["plan", "run"]) - parser.add_argument("--root", type=Path, default=Path(".")) - parser.add_argument("--base", default="origin/main") - parser.add_argument("--race", action="store_true") - parser.add_argument("--output", type=Path) - parser.add_argument("--github-output", type=Path) - parser.add_argument("--summary", type=Path) - args = parser.parse_args() - config = json.loads(CONFIG.read_text()) - root = args.root.resolve() - files = changed_files(root, args.base) - if args.action == "run": - run_unit_tests(root, files, config, args.race) - return - plan = make_plan(files, config) - if args.output: - args.output.write_text(json.dumps(plan, indent=2) + "\n") - if args.github_output: - with args.github_output.open("a") as output: - for key in ("go", "ddto", "evaluator", "integrations", "playwright"): - output.write(key + "=" + json.dumps(plan[key], separators=(",", ":")) + "\n") - if args.summary: - args.summary.write_text(summary(plan)) - print(summary(plan), end="") - - -if __name__ == "__main__": - main() diff --git a/scripts/test_ci_cucumber_matrix.py b/scripts/test_ci_cucumber_matrix.py index 5374739b4..ebf36938c 100644 --- a/scripts/test_ci_cucumber_matrix.py +++ b/scripts/test_ci_cucumber_matrix.py @@ -138,13 +138,15 @@ def test_existing_output_is_not_reused(self): class WorkflowTests(unittest.TestCase): def test_workers_use_the_planned_runtime_and_lockfile_without_resolving_again(self): - workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/ci-full.yml").read_text() + workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/ci.yml").read_text() worker = workflow.split(" testdrive-cucumber-workflow:\n", 1)[1].split("\n testdrive-webdriverio-workflow:", 1)[0] self.assertIn("configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }}", worker) self.assertIn("node-version: ${{ matrix.configuration.node }}", worker) self.assertIn('test "$(npm --version)" =', worker) self.assertIn("name: ci-cucumber-fixtures", worker) - self.assertIn('npm ci --prefix "$RUNNER_TEMP/cucumber-testdrive"', worker) + self.assertIn("uses: ./.github/actions/node-test-fixture", worker) + self.assertIn("directory: ${{ runner.temp }}/cucumber-testdrive", worker) + self.assertNotIn("dependencies:", worker) # Consume the supplied lockfile. self.assertNotIn("npm install", worker) self.assertNotIn("npm view", worker) self.assertIn("-run '${{ matrix.configuration.tests }}'", worker) diff --git a/scripts/test_ci_smoke.py b/scripts/test_ci_smoke.py deleted file mode 100644 index c87e97b61..000000000 --- a/scripts/test_ci_smoke.py +++ /dev/null @@ -1,42 +0,0 @@ -"""A selected smoke must execute its named cases, not pass after filtering them.""" - -import io -import subprocess -import unittest -from unittest.mock import patch - -import ci_smoke - - -class SmokeTests(unittest.TestCase): - def test_smoke_passes_its_runtime_budget_to_go(self): - for minutes in (8, 12, 20): - with self.subTest(minutes=minutes), patch.object(ci_smoke, "run_tests") as run: - ci_smoke.run_smoke({"tests": "^TestSmoke$", "required": ["TestSmoke"], - "timeout_minutes": minutes}) - self.assertIn(f"-timeout={minutes}m", run.call_args.args[0]) - - def test_missing_skipped_or_filtered_cases_cannot_pass(self): - for passed in (set(), {"TestSmoke"}, {"TestSmoke/other"}): - with self.assertRaises(ValueError): - ci_smoke.require_cases(passed, ["TestSmoke/passing"]) - ci_smoke.require_cases({"TestSmoke/passing"}, ["TestSmoke/passing"]) - - def test_json_exit_and_required_case_both_gate_success(self): - class Process: - returncode = 0 - stdout = io.StringIO('{"Action":"pass","Test":"TestSmoke/passing"}\n') - def __enter__(self): return self - def __exit__(self, *args): pass - def wait(self): return self.returncode - with patch.object(ci_smoke.subprocess, "Popen", return_value=Process()): - ci_smoke.run_tests(["go"], {}, ["TestSmoke/passing"]) - process = Process() - process.returncode = 1 - with patch.object(ci_smoke.subprocess, "Popen", return_value=process): - with self.assertRaises(subprocess.CalledProcessError): - ci_smoke.run_tests(["go"], {}, ["TestSmoke/passing"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_ci_test_plan.py b/scripts/test_ci_test_plan.py deleted file mode 100644 index 45e1db69d..000000000 --- a/scripts/test_ci_test_plan.py +++ /dev/null @@ -1,316 +0,0 @@ -"""Verify active PR selection, reverse Go dependencies and required check gates.""" - -import copy -import json -import os -from pathlib import Path -import re -import subprocess -import tempfile -import unittest -from unittest.mock import patch - -import ci_test_plan as planner - - -ROOT = Path(__file__).resolve().parents[1] -CONFIG = json.loads(planner.CONFIG.read_text()) - - -class SelectionTests(unittest.TestCase): - def test_python_change_selects_only_python_smoke(self): - plan = planner.make_plan(["internal/testdrive/python_validation.go"], CONFIG) - self.assertEqual(plan["suites"], ["python"]) - self.assertEqual(plan["integrations"], [{"id": "python", "runtime": "python"}]) - self.assertTrue(plan["go"]) - self.assertFalse(plan["ddto"]) - self.assertEqual(plan["mode"], "fast") - - def test_shared_unknown_build_and_deleted_inputs_select_every_smoke(self): - for path in ("internal/ext/command.go", "internal/onboard/ci_shell_state.go", - "internal/testdrive/jest_features.go", "internal/testdrive/mocha.go", - "internal/testdrive/new_fixture.json", - "go.mod", "go.sum", ".github/workflows/ci.yml"): - with self.subTest(path=path): - plan = planner.make_plan([path], CONFIG) - self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) - self.assertTrue(plan["playwright"]) - - def test_docs_only_and_empty_diff_omit_runtime_checks(self): - for files in ([], ["docs/onboarding.md", "README.md", "ddto/docs/validation.md"]): - plan = planner.make_plan(files, CONFIG) - self.assertFalse(plan["go"]) - self.assertFalse(plan["ddto"]) - self.assertFalse(plan["playwright"]) - self.assertEqual(plan["integrations"], []) - - def test_unknown_history_runs_all_checks(self): - plan = planner.make_plan(None, CONFIG) - self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) - self.assertTrue(plan["go"] and plan["ddto"] and plan["evaluator"]) - - def test_embedded_instructions_and_scripts_are_code_inputs(self): - for path in ("internal/onboard/instructions/pytest.md", "internal/testdrive/scripts/pytest_observer.py"): - plan = planner.make_plan([path], CONFIG) - self.assertIn("python", plan["suites"]) - self.assertTrue(plan["go"]) - - def test_javascript_dependencies_propagate_to_every_javascript_consumer(self): - plan = planner.make_plan(["internal/platform/scripts/javascript_env.js"], CONFIG) - self.assertEqual(set(plan["suites"]), {"javascript", "jest", "vitest", "mocha", "cypress", - "playwright", "cucumber", "webdriverio", "onboarding"}) - - def test_javascript_sources_select_only_javascript_consumers(self): - consumers = {"javascript", "jest", "vitest", "mocha", "cypress", "playwright", - "cucumber", "webdriverio", "onboarding"} - for directory in ("platform", "onboard", "testdrive"): - for name in ("javascript.go", "javascript_language.go"): - path = f"internal/{directory}/{name}" - with self.subTest(path=path): - self.assertEqual(set(planner.make_plan([path], CONFIG)["suites"]), consumers) - - def test_java_sources_still_select_both_build_tool_smokes(self): - for path, suites in ( - ("internal/platform/java.go", {"java"}), - ("internal/onboard/java_ci.go", {"java", "onboarding"}), - ("internal/testdrive/java_validation.go", {"java"}), - ): - with self.subTest(path=path): - plan = planner.make_plan([path], CONFIG) - self.assertEqual(set(plan["suites"]), suites) - self.assertTrue({"java-maven", "java-gradle"}.issubset( - entry["id"] for entry in plan["integrations"])) - - def test_cross_framework_onboarding_helpers_select_all_smokes(self): - for path in ("internal/onboard/jest_command.go", "internal/onboard/python_handwritten.go"): - with self.subTest(path=path): - plan = planner.make_plan([path], CONFIG) - self.assertEqual(set(plan["suites"]), set(CONFIG["suites"])) - - def test_cross_framework_coverage_and_path_helpers_select_their_consumers(self): - for path, expected in ( - ("internal/testdrive/cucumber_coverage.go", set(CONFIG["suites"])), - ("internal/testdrive/cypress_coverage.go", set(CONFIG["suites"])), - ("internal/framework/cypress.go", {"cypress", "playwright"}), - ): - with self.subTest(path=path): - self.assertEqual(set(planner.make_plan([path], CONFIG)["suites"]), expected) - - def test_mocha_runtime_fixture_helper_tracks_mocha_and_webdriverio_smokes(self): - path = "internal/testdrive/mocha_validation_test.go" - self.assertEqual(planner.affected_suites(path, CONFIG), {"mocha", "webdriverio", "onboarding"}) - plan = planner.make_plan([path], CONFIG) - self.assertTrue(plan["go"]) - self.assertEqual({entry["id"] for entry in plan["integrations"]}, - {"mocha", "webdriverio", "onboarding"}) - self.assertFalse(plan["playwright"]) - - def test_runtime_backed_test_edits_select_their_framework_smokes(self): - for path, expected in ( - ("internal/testdrive/vitest_review_test.go", {"vitest", "onboarding"}), - ("internal/testdrive/vitest_test.go", {"vitest", "onboarding"}), - ("internal/testdrive/jest_preflight_test.go", {"jest"}), - ("internal/testdrive/cucumber_support_test.go", {"cucumber", "onboarding"}), - ("internal/framework/webdriverio_test.go", {"webdriverio", "onboarding"}), - ("internal/testdrive/webdriverio_followup_test.go", {"webdriverio", "onboarding"}), - ("internal/testdrive/webdriverio_review_test.go", {"webdriverio", "onboarding"}), - ("internal/testdrive/webdriverio_test.go", {"webdriverio", "onboarding"}), - ("internal/platform/javascript_test.go", {"javascript", "jest", "vitest", "mocha", - "cypress", "playwright", "cucumber", "webdriverio", "onboarding"}), - ): - with self.subTest(path=path): - plan = planner.make_plan([path], CONFIG) - self.assertEqual(set(plan["suites"]), expected) - self.assertTrue(plan["go"]) - - def test_runtime_fixture_gates_cannot_be_classified_as_unit_only(self): - gate = re.compile(r'os\.Getenv\("DDTEST_(?:[A-Z_]+_NODE_MODULES|RUN_[A-Z_]+_INTEGRATION_TEST)"\)') - for path in (ROOT / "internal").rglob("*_test.go"): - if gate.search(path.read_text()): - relative = path.relative_to(ROOT).as_posix() - with self.subTest(path=relative): - plan = planner.make_plan([relative], CONFIG) - self.assertTrue(plan["integrations"] or plan["playwright"]) - - def test_instrumentation_changes_enable_native_module_checks(self): - for path in ("ddto/internal/runner/run.go", "main.go", "go.mod", "internal/cmd/instrument.go"): - self.assertTrue(planner.make_plan([path], CONFIG)["ddto"]) - - def test_fast_workflow_changes_execute_its_conditional_checks(self): - plan = planner.make_plan([".github/workflows/ci.yml"], CONFIG) - self.assertTrue(plan["go"]) - self.assertTrue(plan["ddto"]) - self.assertTrue(plan["evaluator"]) - - def test_instrumentation_module_changes_do_not_run_other_language_frameworks(self): - plan = planner.make_plan(["ddto/internal/runner/run.go"], CONFIG) - self.assertTrue(plan["ddto"] and plan["go"]) - self.assertEqual(plan["integrations"], []) - self.assertFalse(plan["playwright"]) - - def test_evaluator_checks_are_selected_for_evaluator_changes(self): - plan = planner.make_plan(["skills/ddtest-onboarding-eval/scripts/run_eval.py"], CONFIG) - self.assertTrue(plan["evaluator"]) - self.assertFalse(plan["go"]) - self.assertEqual(plan["integrations"], []) - self.assertFalse(planner.make_plan(["internal/testdrive/python_validation.go"], CONFIG)["evaluator"]) - - def test_unit_test_edits_run_go_without_external_framework_smokes(self): - plan = planner.make_plan(["internal/testdrive/jest_test.go"], CONFIG) - self.assertTrue(plan["go"]) - self.assertEqual(plan["integrations"], []) - self.assertFalse(plan["playwright"]) - - -class GoSelectionTests(unittest.TestCase): - def setUp(self): - self.root = Path("/example").resolve() - self.packages = [ - {"Dir": str(self.root / "lib"), "ImportPath": "example/lib"}, - {"Dir": str(self.root / "consumer"), "ImportPath": "example/consumer", "Imports": ["example/lib"]}, - {"Dir": str(self.root / "external"), "ImportPath": "example/external", "XTestImports": ["example/consumer"]}, - {"Dir": str(self.root / "other"), "ImportPath": "example/other"}, - ] - - def test_production_change_reaches_transitive_test_importers(self): - self.assertEqual(planner.select_packages(self.packages, ["lib/lib.go"], self.root), - ["example/consumer", "example/external", "example/lib"]) - - def test_test_only_change_does_not_invalidate_production_consumers(self): - self.assertEqual(planner.select_packages(self.packages, ["lib/lib_test.go"], self.root), ["example/lib"]) - - def test_non_go_inputs_deleted_packages_and_unknown_history_run_all_packages(self): - expected = sorted(package["ImportPath"] for package in self.packages) - for files in (None, ["go.mod"], ["lib/testdata/input.txt"], ["deleted/old.go"], ["ddto/runtime.go"]): - self.assertEqual(planner.select_packages(self.packages, files, self.root), expected) - - def test_go_json_stream_is_decoded_without_compiling_test_binaries(self): - text = "\n" + "\n".join(json.dumps(package) for package in self.packages) - self.assertEqual(list(planner.json_objects(text)), self.packages) - - def test_docs_do_not_start_go(self): - with patch.object(planner.subprocess, "run") as run: - planner.run_unit_tests(self.root, ["README.md"], CONFIG) - run.assert_not_called() - - def test_native_run_keeps_package_selection_race_and_full_package_budget(self): - inventory = subprocess.CompletedProcess([], 0, "\n".join(json.dumps(p) for p in self.packages)) - with patch.object(planner.subprocess, "run", return_value=inventory) as run: - planner.run_unit_tests(self.root, ["lib/lib_test.go"], CONFIG, race=True) - command = run.call_args.args[0] - self.assertEqual(command[:2], ["go", "test"]) - self.assertIn("-race", command) - self.assertEqual([arg for arg in command[2:] if not arg.startswith("-")], ["example/lib"]) - timeout = next(arg for arg in command if arg.startswith("-timeout=")) - package_minutes = int(timeout.removeprefix("-timeout=").removesuffix("m")) - # Runtime-backed tests must fit the same 20-minute package budget as - # the full CI lane, with extra wall time for setup and compilation. - self.assertGreaterEqual(package_minutes, 20) - fast = (ROOT / ".github/workflows/ci.yml").read_text() - unit = fast.split(" unit:\n", 1)[1].split("\n scripts:\n", 1)[0] - job_minutes = int(re.search(r"timeout-minutes: (\d+)", unit).group(1)) - self.assertGreaterEqual(job_minutes, package_minutes + 5) - - -class GitDiffTests(unittest.TestCase): - def test_diff_includes_worktree_deletions_both_sides_of_renames_and_untracked_files(self): - responses = [subprocess.CompletedProcess([], 0, "base\n"), - subprocess.CompletedProcess([], 0, b"old.go\0new.go\0deleted.go\0"), - subprocess.CompletedProcess([], 0, b"untracked.go\0")] - with patch.object(planner.subprocess, "run", side_effect=responses) as run: - self.assertEqual(planner.changed_files(Path('.'), "origin/main"), - ["deleted.go", "new.go", "old.go", "untracked.go"]) - self.assertIn("--no-renames", run.call_args_list[1].args[0]) - - def test_missing_history_cannot_skip_checks(self): - for base in ("", "0" * 40): - self.assertIsNone(planner.changed_files(Path('.'), base)) - with patch.object(planner.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["git"])): - self.assertIsNone(planner.changed_files(Path('.'), "missing")) - - -class WorkflowTests(unittest.TestCase): - def test_full_matrix_is_preserved_and_gates_release(self): - full = (ROOT / ".github/workflows/ci-full.yml").read_text() - jobs = set(re.findall(r"^ ([\w-]+):$", full.split("jobs:\n", 1)[1], re.M)) - for suite in CONFIG["suites"].values(): - self.assertTrue(set(suite["jobs"]).issubset(jobs)) - self.assertIn("uses: ./.github/workflows/ci.yml", full) - self.assertIn("uses: ./.github/workflows/go-instrumentation.yml", full) - release = (ROOT / ".github/workflows/release.yml").read_text() - self.assertIn("uses: ./.github/workflows/ci-full.yml", release) - self.assertIn("needs: compatibility", release) - - def test_fast_workflow_omits_historical_matrix_coverage_and_sdk_weaving(self): - fast = (ROOT / ".github/workflows/ci.yml").read_text() - self.assertNotIn("test-shards:", fast) - self.assertNotIn("Configure Datadog Test Optimization", fast) - self.assertNotIn("-coverpkg=./...", fast) - self.assertNotIn("shadow", fast) - self.assertIn("ci_test_plan.py run", fast) - self.assertIn("ci_smoke.py --id", fast) - self.assertIn(" lint:\n", fast) - - def test_fast_smokes_have_unique_ids_and_explicit_required_cases(self): - entries = [entry for suite in CONFIG["suites"].values() for entry in suite["smoke"]] - self.assertEqual(len(entries), len({entry["id"] for entry in entries})) - for entry in entries: - self.assertTrue(entry["required"]) - self.assertTrue(entry["tests"].startswith("^Test")) - - def test_smokes_preserve_integration_budgets_and_allow_job_setup_time(self): - # These are the existing full lanes' Go test allowances, including Go's - # default ten minutes for the JavaScript platform compatibility tests. - budgets = {"python": 20, "ruby": 20, "java-maven": 20, "java-gradle": 20, - "javascript": 10, "jest": 12, "vitest": 15, "mocha": 10, - "cypress": 12, "playwright": 12, "cucumber": 12, - "webdriverio": 12, "onboarding": 8} - workflow = (ROOT / ".github/workflows/ci.yml").read_text() - integration = workflow.split(" integration:\n", 1)[1].split("\n playwright:\n", 1)[0] - playwright = workflow.split(" playwright:\n", 1)[1].split("\n evaluator-isolation:\n", 1)[0] - for suite in CONFIG["suites"].values(): - for entry in suite["smoke"]: - with self.subTest(smoke=entry["id"]): - minutes = entry["timeout_minutes"] - self.assertGreaterEqual(minutes, budgets[entry["id"]]) - job = playwright if entry["runtime"] == "playwright" else integration - job_minutes = int(re.search(r"timeout-minutes: (\d+)", job).group(1)) - self.assertGreaterEqual(job_minutes, minutes + 5) - - def test_fast_units_provision_and_require_project_runtimes(self): - fast = (ROOT / ".github/workflows/ci.yml").read_text() - unit = fast.split(" unit:\n", 1)[1].split("\n scripts:\n", 1)[0] - for prerequisite in ("actions/setup-node@", "actions/setup-python@", "ruby/setup-ruby@", - "npm install --global yarn@1.22.22", - 'python -m pip install "ddtrace>=4.11.0" pytest pytest-cov pytest-xdist'): - self.assertIn(prerequisite, unit) - self.assertLess(unit.index(prerequisite), unit.index("ci_test_plan.py run")) - check = unit.split("- name: Test affected Go packages and their dependents", 1)[1] - self.assertIn("DDTEST_REQUIRE_PROJECT_RUNTIMES: '1'", check) - - def test_required_test_gate_rejects_failed_cancelled_and_unexpectedly_skipped_checks(self): - workflow = (ROOT / ".github/workflows/ci.yml").read_text().split(" test:\n", 1)[1] - guard = re.search(r"python3 - <<'PY'\n(.*?)\n PY", workflow, re.S).group(1) - code = "\n".join(line[10:] for line in guard.splitlines()) - outputs = {"go": "true", "ddto": "false", "evaluator": "false", "integrations": "[]", "playwright": "false"} - results = {name: {"result": "success"} for name in ("plan", "scripts", "unit", "instrument", "lint")} - results.update({name: {"result": "skipped"} for name in ("integration", "playwright", "instrumentation-unit", "evaluator-isolation")}) - results["plan"]["outputs"] = outputs - def run(value): - return subprocess.run(["python3", "-c", code], capture_output=True, - env={**os.environ, "RESULTS": json.dumps(value)}).returncode - self.assertEqual(run(results), 0) - for name in ("plan", "scripts", "unit", "instrument", "lint"): - for status in ("failure", "cancelled", "skipped"): - broken = copy.deepcopy(results) - broken[name]["result"] = status - self.assertNotEqual(run(broken), 0, (name, status)) - selected = copy.deepcopy(results) - selected["plan"]["outputs"]["integrations"] = '[{"id":"python"}]' - self.assertNotEqual(run(selected), 0) - selected["integration"]["result"] = "success" - self.assertEqual(run(selected), 0) - - -if __name__ == "__main__": - unittest.main() From 2ad0d3249be65a1a2550998ad16d0878b04644af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 16:20:43 +0100 Subject: [PATCH 13/14] Speed up Go CI discovery and refresh compiler caches Isolate crash-recovery helpers and allocation measurements from CI reporting while retaining the original behavioral checks and memory limits. DD_1TLG1W DD_AYZRD6 --- .github/workflows/ci.yml | 21 ++- docs/ci-performance.md | 25 ++++ internal/onboard/ci_provider_jenkins_test.go | 37 +++-- internal/releaseworkflow/ci_shards_test.go | 34 +++++ .../releaseworkflow/release_commit_test.go | 7 +- internal/testdrive/service_owner_test.go | 11 +- scripts/go_test_shards.py | 5 +- scripts/test_go_test_shards.py | 2 +- scripts/testnames/main.go | 132 ++++++++++++++++++ scripts/testnames/main_test.go | 122 ++++++++++++++++ 10 files changed, 376 insertions(+), 20 deletions(-) create mode 100644 scripts/testnames/main.go create mode 100644 scripts/testnames/main_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99f3e5aeb..4262483f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,11 +81,13 @@ jobs: shard: [1, 2, 3] env: DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + ORCHESTRION_VERSION: 5df280060f05cbf2863d43748667fcf9cd18c448 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go + id: go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -117,9 +119,13 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + ~/go/bin/orchestrion + # Cache entries are immutable: a new revision must save fresh compiler + # output, including the SDK install and instrumented test builds. + key: ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- + ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}- - name: Download dependencies run: go mod download @@ -155,9 +161,12 @@ jobs: uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 with: languages: go + # The compiler cache above already owns modules and Orchestrion. + # Avoid restoring the action's overlapping module cache a second time. + cache: false # Pin upstream Go 1.27 compatibility fixes until they reach a stable # Orchestrion release. - go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 + go-tracer-version: ${{ env.ORCHESTRION_VERSION }} api_key: ${{ secrets.DATADOG_API_KEY }} site: datadoghq.com service: ddtest @@ -226,6 +235,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go + id: go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -258,9 +268,10 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- + ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}- - name: Download dependencies run: go mod download diff --git a/docs/ci-performance.md b/docs/ci-performance.md index ce1039f63..9cef13736 100644 --- a/docs/ci-performance.md +++ b/docs/ci-performance.md @@ -5,6 +5,23 @@ testdrive scenarios. Both existing Go instrumentation paths and their coverage remain enabled. Datadog Test Impact Analysis owns test skipping; this workflow does not maintain a separate changed-file test selector. +## Go test discovery and compiler caches + +The Go shard selector runs `go run ./scripts/testnames`. The helper asks +`go list -race -json ./...` for the files selected by the current platform and +build tags, then uses Go's parser and example discovery to list tests, examples +and fuzz targets. Discovery builds only the small helper; it does not build +every test executable or run their initializers and `TestMain` functions. + +The native Orchestrion and `ddtest instrument` jobs retain separate compiler +caches. Keys include OS, architecture, Go version, dependencies and the source +revision; the Orchestrion cache also includes its pinned version and executable. +A new revision restores compatible previous output and saves updated compiler +artifacts, rather than repeatedly restoring an immutable dependency-only cache. +The Datadog action uses this cache instead of restoring another overlapping +module cache. Its configuration script still runs to resolve compatible SDK +versions and set up the current job's instrumentation and reporting. + ## Shared CLI executable The `test-binary` job builds one static executable for the Linux public CLI journeys, @@ -47,3 +64,11 @@ it only imports dependencies and checks their versions. The actual testdrive journeys keep their original reporting, local intake, configuration variants and six feature assertions. No validation scenario is removed or moved to a release only workflow. + +## Test helper isolation + +Crash-recovery helpers run with their own CI reporting disabled, while their +parent tests keep reporting and checking process ownership and cleanup. Jenkins +allocation budgets run in an isolated helper so background SDK allocations do +not count against the parser. The original memory limits remain, and the parent +also exercises every behavioral assertion to retain instrumentation coverage. diff --git a/internal/onboard/ci_provider_jenkins_test.go b/internal/onboard/ci_provider_jenkins_test.go index 73359deab..eb176af22 100644 --- a/internal/onboard/ci_provider_jenkins_test.go +++ b/internal/onboard/ci_provider_jenkins_test.go @@ -11,6 +11,7 @@ import ( "errors" "maps" "os" + "os/exec" "path/filepath" "runtime" "slices" @@ -1388,6 +1389,18 @@ func TestJenkinsRubyVersionManagersKeepTheStatedRuntime(t *testing.T) { // cells of all of a Jenkinsfile's matrices, and a value that interpolates // others. func TestJenkinsExpansionsAreBounded(t *testing.T) { + isolated := os.Getenv("DDTEST_JENKINS_ALLOCATION_HELPER") == "1" + if isolated { + require.Equal(t, "false", os.Getenv("DD_CIVISIBILITY_ENABLED")) + } else { + // TotalAlloc is process-wide. Measure without background SDK work, + // then also run the behavioral assertions here to retain the + // instrumented parent's coverage of the parser's bounds. + command := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestJenkinsExpansionsAreBounded$") + command.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_JENKINS_ALLOCATION_HELPER=1") + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + } values := func(n int) string { var items []string for i := range n { @@ -1400,13 +1413,17 @@ func TestJenkinsExpansionsAreBounded(t *testing.T) { } huge := "pipeline {\n agent any\n stages {\n" + matrix("M", " axis { name 'A'; values "+values(200)+" }\n axis { name 'B'; values "+values(1000)+" }\n", "stage('t') { steps { sh 'npm test' } }") + " }\n}\n" var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) + if isolated { + runtime.GC() + runtime.ReadMemStats(&before) + } workflow, err := readJenkinsfile(t.TempDir(), "Jenkinsfile", huge) - runtime.ReadMemStats(&after) + if isolated { + runtime.ReadMemStats(&after) + require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(8<<20), "200,000 cells are never built") + } require.NoError(t, err) require.Equal(t, []string{"UNRESOLVED Jenkinsfile M: ddtest cannot expand its matrix (it has more than 200 cells), so it cannot tell which stages it runs"}, jenkinsRuns(workflow.Jobs["M"])) - require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(8<<20), "200,000 cells are never built") // Nested matrices multiply: past the Jenkinsfile's budget, each is // unresolved. @@ -1432,15 +1449,19 @@ func TestJenkinsExpansionsAreBounded(t *testing.T) { doubling += "def v" + strconv.Itoa(i) + " = \"${v" + strconv.Itoa(i-1) + "}${v" + strconv.Itoa(i-1) + "}\"\n" } doubling += "node { stage('T') { sh \"echo ${v30}\"\nsh 'npm test' } }\n" - runtime.GC() - runtime.ReadMemStats(&before) + if isolated { + runtime.GC() + runtime.ReadMemStats(&before) + } jobs := jenkinsJobs(t, doubling) - runtime.ReadMemStats(&after) + if isolated { + runtime.ReadMemStats(&after) + require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(16<<20), "a 16 GiB string is never built") + } runs := jenkinsRuns(jobs["T"]) require.Len(t, runs, 2) require.Contains(t, runs[0], "UNRESOLVED Jenkinsfile T: its sh script reads ${v30}, which only the Jenkins build knows") require.Equal(t, "npm test", runs[1]) - require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(16<<20), "a 16 GiB string is never built") // Nested stages repeat their names in the jobs inside: a long one is // named by its line. diff --git a/internal/releaseworkflow/ci_shards_test.go b/internal/releaseworkflow/ci_shards_test.go index 1d357ed66..026f91102 100644 --- a/internal/releaseworkflow/ci_shards_test.go +++ b/internal/releaseworkflow/ci_shards_test.go @@ -15,16 +15,50 @@ type ciTestJob struct { If string Needs any Uses string + Env map[string]string Strategy struct { FailFast bool `yaml:"fail-fast"` Matrix struct{ Shard []any } } Steps []struct { Name, Run, Uses string + ID string `yaml:"id"` Env map[string]string + With map[string]string } } +func TestCompilerCachesRefreshAndKeepInstrumentationSeparate(t *testing.T) { + jobs := ciTestJobs(t) + keys := make(map[string]string) + for _, name := range []string{"test-shards", "test-ddtest-instrument-shards"} { + job := jobs[name] + for _, step := range job.Steps { + if step.Name == "Set up Go" { + require.Equal(t, "go", step.ID) + require.Equal(t, "false", step.With["cache"]) + } + if step.Name == "Cache Go modules" { + key := step.With["key"] + for _, input := range []string{"runner.os", "runner.arch", "steps.go.outputs.go-version", "github.sha"} { + require.Contains(t, key, "${{ "+input+" }}", name) + } + require.Contains(t, step.With["path"], "~/.cache/go-build") + require.NotEmpty(t, step.With["restore-keys"]) + require.Equal(t, strings.TrimSuffix(key, "${{ github.sha }}"), strings.Split(strings.TrimSpace(step.With["restore-keys"]), "\n")[0]) + keys[name] = key + } + if strings.HasPrefix(step.Uses, "datadog/test-visibility-github-action@") { + require.Equal(t, "false", step.With["cache"], "avoid a second overlapping module-cache restore") + require.Equal(t, "${{ env.ORCHESTRION_VERSION }}", step.With["go-tracer-version"]) + require.NotEmpty(t, job.Env["ORCHESTRION_VERSION"]) + } + } + require.NotEmpty(t, keys[name]) + } + require.NotEqual(t, keys["test-shards"], keys["test-ddtest-instrument-shards"]) +} + func TestReleaseUsesTheSameCIAndRetainsTheInstrumentationGate(t *testing.T) { data, err := os.ReadFile("../../.github/workflows/release.yml") require.NoError(t, err) diff --git a/internal/releaseworkflow/release_commit_test.go b/internal/releaseworkflow/release_commit_test.go index 64aa27dad..909477c0b 100644 --- a/internal/releaseworkflow/release_commit_test.go +++ b/internal/releaseworkflow/release_commit_test.go @@ -22,9 +22,12 @@ func TestReleaseCommit(t *testing.T) { } git := func(dir string, args ...string) string { t.Helper() - out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).Output() + command := exec.Command("git", append([]string{"-C", dir}, args...)...) + var diagnostic bytes.Buffer + command.Stderr = &diagnostic + out, err := command.Output() if err != nil { - t.Fatalf("git %s: %v", strings.Join(args, " "), err) + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, diagnostic.String()) } return strings.TrimSpace(string(out)) } diff --git a/internal/testdrive/service_owner_test.go b/internal/testdrive/service_owner_test.go index 4366a7de8..4ed3fdfb0 100644 --- a/internal/testdrive/service_owner_test.go +++ b/internal/testdrive/service_owner_test.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/ddtest/internal/framework" "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,6 +29,7 @@ type crashedService struct { func TestDeclaredServiceStopsWhenCoordinatorIsKilled(t *testing.T) { if root := os.Getenv("DDTEST_SERVICE_CRASH_ROOT"); root != "" { + require.Equal(t, "false", os.Getenv("DD_CIVISIBILITY_ENABLED"), "the helper must not start a CI test session") for _, key := range []string{"TMPDIR", "TMP", "TEMP"} { t.Setenv(key, os.Getenv("DDTEST_SERVICE_CRASH_TMP")) } @@ -69,15 +71,20 @@ func TestDeclaredServiceStopsWhenCoordinatorIsKilled(t *testing.T) { defer func() { _ = canary.Close() }() binary, err := os.Executable() require.NoError(t, err) + t.Setenv("DD_CIVISIBILITY_ENABLED", "true") // Exercise helper isolation even in a plain local test run. command := exec.Command(binary, "-test.run=^TestDeclaredServiceStopsWhenCoordinatorIsKilled$") - command.Env = append(os.Environ(), "DDTEST_SERVICE_CRASH_ROOT="+root, "DDTEST_SERVICE_CRASH_TMP="+temporary) + command.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_SERVICE_CRASH_ROOT="+root, "DDTEST_SERVICE_CRASH_TMP="+temporary) var diagnostic bytes.Buffer command.Stdout = &diagnostic command.Stderr = &diagnostic require.NoError(t, command.Start()) t.Cleanup(func() { _ = command.Process.Kill() }) ready := filepath.Join(root, "ready.json") - require.Eventually(t, func() bool { _, err := os.Stat(ready); return err == nil }, 10*time.Second, 20*time.Millisecond) + if !assert.Eventually(t, func() bool { _, err := os.Stat(ready); return err == nil }, 10*time.Second, 20*time.Millisecond) { + _ = command.Process.Kill() + _ = command.Wait() + t.Fatalf("service helper did not become ready:\n%s", diagnostic.String()) + } data, err := os.ReadFile(ready) require.NoError(t, err) var state crashedService diff --git a/scripts/go_test_shards.py b/scripts/go_test_shards.py index b2009c57b..41755e96b 100644 --- a/scripts/go_test_shards.py +++ b/scripts/go_test_shards.py @@ -19,8 +19,9 @@ def parse_shard(value): def discover_tests(): - # Match CI's build selection, including tests guarded by the race build tag. - result = subprocess.run(["go", "test", "-race", "-list", ".", "./..."], + # Go selects race-enabled source files; the helper parses declarations + # without compiling every test executable or running package initializers. + result = subprocess.run(["go", "run", "./scripts/testnames"], check=True, capture_output=True, text=True) names = sorted({line for line in result.stdout.splitlines() if re.fullmatch(r"(?:Test|Example|Fuzz)\w*", line)}) diff --git a/scripts/test_go_test_shards.py b/scripts/test_go_test_shards.py index 2728e52fa..61b4834c5 100644 --- a/scripts/test_go_test_shards.py +++ b/scripts/test_go_test_shards.py @@ -22,7 +22,7 @@ def test_discovery_uses_go_build_selection_and_ignores_diagnostics_and_benchmark result = subprocess.CompletedProcess([], 0, "TestCase\nTestCase\nExample\nFuzzSeed\nBenchmarkOther\nok\tpkg\n") with patch.object(subprocess, "run", return_value=result) as run: self.assertEqual(shards.discover_tests(), ["Example", "FuzzSeed", "TestCase"]) - self.assertEqual(run.call_args.args[0], ["go", "test", "-race", "-list", ".", "./..."]) + self.assertEqual(run.call_args.args[0], ["go", "run", "./scripts/testnames"]) def test_failed_or_empty_discovery_cannot_silently_pass(self): with patch.object(subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["go"])): diff --git a/scripts/testnames/main.go b/scripts/testnames/main.go new file mode 100644 index 000000000..e7e6ec2ea --- /dev/null +++ b/scripts/testnames/main.go @@ -0,0 +1,132 @@ +// Command testnames lists the root module's race-enabled tests without building +// test executables or running package initializers and TestMain. +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/token" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "unicode" + "unicode/utf8" +) + +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { + return true + } + r, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(r) +} + +// Match cmd/go's syntactic signature check, including aliased and dot imports. +func hasArgument(fn *ast.FuncDecl, kind string) bool { + if fn.Type.Results.NumFields() != 0 || fn.Type.Params.NumFields() != 1 || fn.Type.TypeParams.NumFields() != 0 { + return false + } + pointer, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr) + if !ok { + return false + } + switch argument := pointer.X.(type) { + case *ast.Ident: + return argument.Name == kind + case *ast.SelectorExpr: + return argument.Sel.Name == kind + } + return false +} + +func fileTests(path string) ([]string, error) { + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + return nil, err + } + var names []string + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok || fn.Recv != nil { + continue + } + name := fn.Name.Name + if name == "TestMain" && hasArgument(fn, "M") { + continue + } + kind := "" + switch { + case isTest(name, "Test"): + kind = "T" + case isTest(name, "Fuzz"): + kind = "F" + } + if kind != "" { + if !hasArgument(fn, kind) { + return nil, fmt.Errorf("%s: invalid signature for %s", path, name) + } + names = append(names, name) + } + } + for _, example := range doc.Examples(file) { + if example.Output != "" || example.EmptyOutput { + names = append(names, "Example"+example.Name) + } + } + return names, nil +} + +func discover(ctx context.Context, directory string) ([]string, error) { + // Let Go select files for the current platform, tags, cgo and race mode. + command := exec.CommandContext(ctx, "go", "list", "-race", "-json", "./...") + command.Dir = directory + command.Stderr = os.Stderr + output, err := command.Output() + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(output)) + var names []string + for { + var pkg struct { + Dir string + TestGoFiles, XTestGoFiles []string + } + if err := decoder.Decode(&pkg); err == io.EOF { + break + } else if err != nil { + return nil, err + } + for _, path := range append(pkg.TestGoFiles, pkg.XTestGoFiles...) { + selected, err := fileTests(filepath.Join(pkg.Dir, path)) + if err != nil { + return nil, err + } + names = append(names, selected...) + } + } + slices.Sort(names) + return slices.Compact(names), nil +} + +func main() { + names, err := discover(context.Background(), "") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, name := range names { + fmt.Println(name) + } +} diff --git a/scripts/testnames/main_test.go b/scripts/testnames/main_test.go new file mode 100644 index 000000000..85e3e9d93 --- /dev/null +++ b/scripts/testnames/main_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func fixture(t *testing.T, files map[string]string) string { + t.Helper() + t.Setenv("GOFLAGS", "") // The reference fixture must use the native compiler. + t.Setenv("GOWORK", "off") + root := t.TempDir() + files["go.mod"] = "module example.com/inventory\n\ngo 1.24\n" + for path, source := range files { + require.NoError(t, os.WriteFile(filepath.Join(root, path), []byte(source), 0600)) + } + return root +} + +func TestSourceInventoryMatchesGoTest(t *testing.T) { + otherOS := "windows" + if runtime.GOOS == otherOS { + otherOS = "linux" + } + root := fixture(t, map[string]string{ + "inventory.go": "package inventory\nfunc Value() {}\nfunc WithoutOutput() {}\ntype Widget struct{}\n", + "inventory_test.go": `package inventory +import ("fmt"; "os"; tst "testing") +func TestMain(m *tst.M) { os.Exit(m.Run()) } +func Test(t *tst.T) {}; func TestSecond(t *tst.T) {} +func TestÉ(t *tst.T) {} +func Testlowercase(t *tst.T) {} +func (Widget) TestMethod(t *tst.T) {} +func BenchmarkIgnored(b *tst.B) {} +func FuzzValue(f *tst.F) { f.Add("seed"); f.Fuzz(func(t *tst.T, s string) {}) } +func ExampleValue() { + fmt.Println("value") + // Output: value +} +func ExampleValue_empty() { + // Output: +} +func ExampleWithoutOutput() {} +var decoy = "func TestNotADeclaration(t *testing.T) {}" +`, + "external_test.go": `package inventory_test +import . "testing" +func TestExternal(t *T) {} +func TestMain(t *T) {} +`, + "race_test.go": "//go:build race\n\npackage inventory\nimport \"testing\"\nfunc TestRace(t *testing.T) {}\n", + "native_test.go": "//go:build !race\n\npackage inventory\nimport \"testing\"\nfunc TestNotRace(t *testing.T) {}\n", + "tagged_test.go": "//go:build ddtest_extra_fixture\n\npackage inventory\nimport \"testing\"\nfunc TestTagged(t *testing.T) {}\n", + "platform_" + otherOS + "_test.go": "package inventory\nimport \"testing\"\nfunc TestOtherOS(t *testing.T) {}\n", + }) + for _, tags := range []string{"", "-tags=ddtest_extra_fixture"} { + t.Run(tags, func(t *testing.T) { + t.Setenv("GOFLAGS", tags) + names, err := discover(t.Context(), root) + require.NoError(t, err) + // Vet warns about the intentionally ignored Testlowercase declaration. + command := exec.CommandContext(t.Context(), "go", "test", "-vet=off", "-race", "-list", ".", "./...") + command.Dir = root + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + var compiled []string + for _, line := range strings.Split(string(output), "\n") { + if len(strings.Fields(line)) == 1 && (strings.HasPrefix(line, "Test") || strings.HasPrefix(line, "Fuzz") || strings.HasPrefix(line, "Example")) { + compiled = append(compiled, line) + } + } + slices.Sort(compiled) + require.Equal(t, compiled, names) + require.Contains(t, names, "TestRace") + require.Contains(t, names, "TestMain") + require.Contains(t, names, "ExampleValue_empty") + require.NotContains(t, names, "TestNotRace") + require.NotContains(t, names, "TestOtherOS") + require.Equal(t, tags != "", slices.Contains(names, "TestTagged")) + }) + } +} + +func TestDiscoveryDoesNotBuildOrRunTheTestBinary(t *testing.T) { + root := fixture(t, map[string]string{ + "inventory.go": "package inventory\nfunc broken() { notDefined() }\nfunc init() { panic(\"initializer ran\") }\n", + "inventory_test.go": "package inventory\nimport \"testing\"\nfunc TestMain(m *testing.M) { panic(\"TestMain ran\") }\nfunc TestSelected(t *testing.T) {}\n", + }) + names, err := discover(t.Context(), root) + require.NoError(t, err) + require.Equal(t, []string{"TestSelected"}, names) +} + +func TestInvalidTestDeclarationsFailDiscovery(t *testing.T) { + for _, declaration := range []string{ + "func TestMissingArgument() {}", + "func TestWrongArgument(t *testing.B) {}", + "func TestResult(t *testing.T) int { return 0 }", + "func TestGeneric[T any](t *testing.T) {}", + "func FuzzWrongArgument(t *testing.T) {}", + } { + t.Run(declaration, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "invalid_test.go") + require.NoError(t, os.WriteFile(path, []byte("package inventory\nimport \"testing\"\n"+declaration), 0600)) + _, err := fileTests(path) + require.ErrorContains(t, err, "invalid signature") + }) + } +} + +func TestGoListFailureCannotProduceAnEmptySuccessfulInventory(t *testing.T) { + root := fixture(t, map[string]string{"inventory.go": "package inventory\nimport \"unterminated\n"}) + _, err := discover(t.Context(), root) + require.Error(t, err) +} From fdf392efdab37ae2fcbf56830b6a96d92ee601d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 17:41:45 +0100 Subject: [PATCH 14/14] Keep the shared CLI in process-owned temporary storage --- internal/testdrive/temp_main_test.go | 2 ++ internal/testutil/binary.go | 21 ++++++++++++++------- internal/testutil/binary_test.go | 8 ++++++-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/internal/testdrive/temp_main_test.go b/internal/testdrive/temp_main_test.go index ea9bb08d3..e5e978ae5 100644 --- a/internal/testdrive/temp_main_test.go +++ b/internal/testdrive/temp_main_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" ) func TestMain(m *testing.M) { @@ -21,6 +22,7 @@ func TestMain(m *testing.M) { os.Exit(1) } _ = os.Setenv("DDTEST_ISOLATED_TEST_TMP", "1") + testutil.SetBinaryDirectory(root) for _, key := range []string{"TMPDIR", "TMP", "TEMP"} { _ = os.Setenv(key, root) } diff --git a/internal/testutil/binary.go b/internal/testutil/binary.go index ed3161667..f0e71946d 100644 --- a/internal/testutil/binary.go +++ b/internal/testutil/binary.go @@ -13,12 +13,19 @@ import ( ) type binaryCache struct { - once sync.Once - path string - err error + once sync.Once + directory string + path string + err error } -var compiled = new(binaryCache) +var compiled = &binaryCache{directory: os.TempDir()} + +// SetBinaryDirectory gives the shared executable a process-owned directory. +// Call from TestMain before running tests that override their temporary paths. +func SetBinaryDirectory(directory string) { + compiled = &binaryCache{directory: directory} +} // Capture the package directory before customer fixtures use t.Chdir. var repositoryRoot, repositoryRootError = filepath.Abs("../..") @@ -34,18 +41,18 @@ func DDTest(t testing.TB) string { } return path } - compiled.once.Do(func() { compiled.path, compiled.err = buildDDTest(t.Context()) }) + compiled.once.Do(func() { compiled.path, compiled.err = buildDDTest(t.Context(), compiled.directory) }) if compiled.err != nil { t.Fatal(compiled.err) } return compiled.path } -func buildDDTest(ctx context.Context) (string, error) { +func buildDDTest(ctx context.Context, parent string) (string, error) { if repositoryRootError != nil { return "", repositoryRootError } - directory, err := os.MkdirTemp("", "ddtest-binary-") + directory, err := os.MkdirTemp(parent, "ddtest-binary-") if err != nil { return "", err } diff --git a/internal/testutil/binary_test.go b/internal/testutil/binary_test.go index 9733fece4..3aea65939 100644 --- a/internal/testutil/binary_test.go +++ b/internal/testutil/binary_test.go @@ -15,11 +15,12 @@ func TestDDTestBuildIsSharedAcrossWorkingDirectories(t *testing.T) { t.Skip("POSIX fake Go executable") } previous := compiled - compiled = new(binaryCache) t.Cleanup(func() { compiled = previous }) t.Setenv("DDTEST_TEST_BINARY", "") directory := t.TempDir() - t.Setenv("TMPDIR", directory) + SetBinaryDirectory(directory) + firstTemporaryDirectory := t.TempDir() + t.Setenv("TMPDIR", firstTemporaryDirectory) log := filepath.Join(directory, "builds") t.Setenv("BUILD_LOG", log) goCommand := "#!/bin/sh\nprintf '%s\\n' \"$PWD\" >> \"$BUILD_LOG\"\nprintf '#!/bin/sh\\nexit 0\\n' > \"$3\"\nchmod +x \"$3\"\n" @@ -27,6 +28,9 @@ func TestDDTestBuildIsSharedAcrossWorkingDirectories(t *testing.T) { t.Setenv("PATH", directory+string(os.PathListSeparator)+os.Getenv("PATH")) t.Chdir(t.TempDir()) first := DDTest(t) + t.Chdir(directory) + require.NoError(t, os.RemoveAll(firstTemporaryDirectory), "simulate the first journey's cleanup") + t.Setenv("TMPDIR", directory) t.Chdir(t.TempDir()) require.Equal(t, first, DDTest(t)) data, err := os.ReadFile(log)