From b275b1615928b15e304e6cca4e5f6b4a0aa613d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 15:42:05 +0100 Subject: [PATCH 1/5] Refactor shared onboarding and testdrive CI handling --- internal/cmd/ci_provider.go | 23 + internal/cmd/onboard.go | 16 +- internal/cmd/testdrive.go | 14 +- internal/onboard/README.md | 43 + internal/onboard/ci_provider.go | 809 ------------------ internal/onboard/ci_provider_buildkite.go | 9 - internal/onboard/ci_provider_circleci.go | 9 - internal/onboard/ci_provider_github.go | 78 +- .../ci_provider_github_test.go} | 41 +- internal/onboard/ci_provider_gitlab.go | 9 - internal/onboard/ci_provider_gitlab_test.go | 4 +- internal/onboard/ci_provider_jenkins.go | 9 - internal/onboard/ci_provider_test.go | 2 +- internal/onboard/ci_selection.go | 240 ++++++ internal/onboard/ci_setup.go | 503 +++++++++++ internal/onboard/ci_statements.go | 21 - internal/onboard/ci_workflows.go | 159 ++++ internal/onboard/discovery.go | 25 - internal/onboard/onboard.go | 2 +- internal/onboard/python.go | 6 +- internal/onboard/ruby.go | 6 +- internal/onboard/scripts.go | 4 +- .../match.go} | 137 +-- internal/testcommand/match_test.go | 59 ++ .../testdrive/{onboard.go => workflow.go} | 433 +--------- internal/testdrive/workflow_commands.go | 86 ++ internal/testdrive/workflow_inputs.go | 63 ++ internal/testdrive/workflow_report.go | 247 ++++++ internal/testdrive/workflow_tracer.go | 91 ++ 29 files changed, 1653 insertions(+), 1495 deletions(-) create mode 100644 internal/onboard/README.md rename internal/{environment/github_workflows_test.go => onboard/ci_provider_github_test.go} (76%) create mode 100644 internal/onboard/ci_selection.go create mode 100644 internal/onboard/ci_setup.go create mode 100644 internal/onboard/ci_workflows.go rename internal/{environment/github_workflows.go => testcommand/match.go} (63%) create mode 100644 internal/testcommand/match_test.go rename internal/testdrive/{onboard.go => workflow.go} (74%) create mode 100644 internal/testdrive/workflow_commands.go create mode 100644 internal/testdrive/workflow_report.go create mode 100644 internal/testdrive/workflow_tracer.go diff --git a/internal/cmd/ci_provider.go b/internal/cmd/ci_provider.go index 6e630d9aa..2919fa5bd 100644 --- a/internal/cmd/ci_provider.go +++ b/internal/cmd/ci_provider.go @@ -22,6 +22,29 @@ import ( // selectCIProvider resolves the CI provider; tests replace it. var selectCIProvider = onboard.SelectCIProvider +// ciOptions keeps the shared CI flags and their resolution order together. +// Each command owns an instance so flag values never leak between commands. +type ciOptions struct { + provider string + runtimes []string + facts []string +} + +func (options *ciOptions) addFlags(command *cobra.Command) { + command.Flags().StringVar(&options.provider, "ci-provider", "", ciProviderFlagUsage()) + ciStatementFlags(command, &options.runtimes, &options.facts) +} + +func (options ciOptions) resolve(command *cobra.Command, structured bool) error { + if err := resolveCIProvider(command, options.provider); err != nil { + return ciProviderFailure(command, structured, err) + } + if err := resolveCIStatements(options.runtimes, options.facts); err != nil { + return ciStatementFailure(command, structured, err) + } + return nil +} + // ciProviderFlagUsage describes --ci-provider on onboard and testdrive. func ciProviderFlagUsage() string { return "CI provider whose configuration to discover, set up and validate (" + strings.Join(onboard.CIProviderNames(), ", ") + "); required when several contain tests" diff --git a/internal/cmd/onboard.go b/internal/cmd/onboard.go index c729fcbf1..bea14ced1 100644 --- a/internal/cmd/onboard.go +++ b/internal/cmd/onboard.go @@ -22,10 +22,10 @@ var onboardCmd = newOnboardCommand(nil) func newOnboardCommand(run func(io.Writer) error) *cobra.Command { var apply, yes, instructions, allowDirty, agent bool - var version, facts, site, format, resume, ciProvider string + var version, facts, site, format, resume string var timeout time.Duration var reviewed, substitute bool - var ciRuntimes, ciFacts []string + var ci ciOptions command := &cobra.Command{ Use: "onboard", Short: "Start here: onboard this repository to Test Optimization", @@ -52,12 +52,9 @@ func newOnboardCommand(run func(io.Writer) error) *cobra.Command { if apply && instructions { return fail("invalid_options", fmt.Errorf("--apply and --instructions are mutually exclusive")) } - if err := resolveCIProvider(cmd, ciProvider); err != nil { - // Without --instructions, onboarding answers non-terminals in the agent protocol. - return ciProviderFailure(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout())), err) - } - if err := resolveCIStatements(ciRuntimes, ciFacts); err != nil { - return ciStatementFailure(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout())), err) + // Without --instructions, onboarding answers non-terminals in the agent protocol. + if err := ci.resolve(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout()))); err != nil { + return err } // start returns the onboarding handoff: the next command, or for a // language whose CI setup the agent applies, the setup to apply first. @@ -147,8 +144,7 @@ func newOnboardCommand(run func(io.Writer) error) *cobra.Command { command.Flags().StringVar(&facts, "facts", "", "JSON build/service facts bound to input_fingerprint; must be outside the checkout") command.Flags().StringVar(&site, "site", "", "Datadog site for a new action (default DD_SITE, otherwise datadoghq.com)") command.Flags().StringVar(&format, "format", "agent", "Result rendering: agent or json for direct host consumption") - command.Flags().StringVar(&ciProvider, "ci-provider", "", ciProviderFlagUsage()) - ciStatementFlags(command, &ciRuntimes, &ciFacts) + ci.addFlags(command) command.Flags().BoolVar(&allowDirty, "allow-dirty", false, "Explicitly extend uncommitted target-file edits after showing their exact diff") agentHelp(command) diff --git a/internal/cmd/testdrive.go b/internal/cmd/testdrive.go index 88f55d159..bb2dbafc8 100644 --- a/internal/cmd/testdrive.go +++ b/internal/cmd/testdrive.go @@ -35,10 +35,10 @@ func newTestdriveCommand() *cobra.Command { return nil }, } - var version, facts, format, resume, detail, site, ciProvider string + var version, facts, format, resume, detail, site string var checkOnly, agent, report bool var all bool - var ciRuntimes, ciFacts []string + var ci ciOptions var allowReviewedBuilds, acceptSubstituteRuntime bool var timeout time.Duration command.Flags().DurationVar(&timeout, "timeout", 10*time.Minute, "Maximum duration of each subprocess, including each full test-suite run") @@ -55,8 +55,7 @@ func newTestdriveCommand() *cobra.Command { command.Flags().StringVar(&resume, "resume", "", "Resume the same repository/framework/scope; tests always execute fresh") command.Flags().StringVar(&facts, "facts", "", "JSON build/service facts bound to input_fingerprint (requires --all)") command.Flags().StringVar(&site, "site", "", "Datadog site retained in the resolved validation plan") - command.Flags().StringVar(&ciProvider, "ci-provider", "", ciProviderFlagUsage()) - ciStatementFlags(command, &ciRuntimes, &ciFacts) + ci.addFlags(command) command.RunE = func(cmd *cobra.Command, _ []string) error { structured := agent || cmd.Flags().Changed("format") @@ -83,11 +82,8 @@ func newTestdriveCommand() *cobra.Command { return fail("invalid_options", fmt.Errorf("--report cannot be combined with --all or --check-only")) } // Reports are bound to the provider they validated, so --report resolves it too. - if err := resolveCIProvider(cmd, ciProvider); err != nil { - return ciProviderFailure(cmd, structured, err) - } - if err := resolveCIStatements(ciRuntimes, ciFacts); err != nil { - return ciStatementFailure(cmd, structured, err) + if err := ci.resolve(cmd, structured); err != nil { + return err } if report { cmd.SilenceErrors = true diff --git a/internal/onboard/README.md b/internal/onboard/README.md new file mode 100644 index 000000000..9182695a1 --- /dev/null +++ b/internal/onboard/README.md @@ -0,0 +1,43 @@ +# Onboarding and testdrive code map + +`ddtest onboard` and `ddtest testdrive` share CI discovery and validation. +Start at the boundary that owns the behavior you need to change: + +| Responsibility | Entry points | +| --- | --- | +| CLI flags, consent and output mode | `internal/cmd/onboard.go`, `internal/cmd/testdrive.go` | +| Shared CI flags and error rendering | `internal/cmd/ci_provider.go` (`ciOptions`) | +| CI provider selection and follow-up arguments | `internal/onboard/ci_selection.go` | +| Provider contract and registrations | `internal/onboard/ci_provider.go`, `ci_providers.go` | +| Workflow model, loading and shared test/setup classification | `internal/onboard/ci_workflows.go` | +| Provider configuration syntax and edits | `internal/onboard/ci_provider_*.go` | +| Shared setup model, support checks and setup wording | `internal/onboard/ci_setup.go` | +| User statements about CI runtimes and server configuration | `internal/onboard/ci_statements.go` | +| Language-specific discovery and setup | `internal/onboard/language.go`, `languages.go`, `*.go` | +| Resolved test commands and validation scope | `internal/onboard/discovery.go`, `scope.go`, `scripts.go` | +| Literal command recognition without executing repository code | `internal/testcommand` | +| Setup plans and bounded source edits | `internal/onboard/execution_plan.go`, `apply*.go` | +| Validation preparation and execution | `internal/testdrive/testdrive.go` | +| Coordinated setup and validation | `internal/testdrive/workflow.go` | +| Restart and resume commands | `internal/testdrive/workflow_commands.go` | +| Workflow tracer selection and metadata preflight | `internal/testdrive/workflow_tracer.go` | +| Saved workflow reports and blocker classification | `internal/testdrive/workflow_report.go` | +| Input fingerprints and accepted setup facts | `internal/testdrive/workflow_inputs.go` | +| Evidence, cleanup and checkpoints | `internal/testdrive/workflow_checks.go`, `workflow_checkpoint.go`, `workflow_receipt.go` | + +Unqualified filenames in a row are in the same directory as that row's first file. + +Providers parse their own syntax into `ciWorkflow`, `runtimeJob` and +`runtimeStep`. Shared discovery, setup and runtime checks consume those models; +language decisions stay behind `discoveryLanguage`. Providers normally use +`findTestWorkflows`' shared classification. GitHub manual instructions preserve +their conservative literal-command inspection and use the same GitHub file +loader as full discovery; full discovery additionally expands composite actions. + +`internal/environment` detects the running CI environment and supplies runtime +tags and runner configuration. Repository CI inspection belongs here, in +`internal/onboard`, and is shared with testdrive. + +After changing these paths, run `make test` and `make lint`. The end-to-end +Codex catalog is described in +[`skills/ddtest-onboarding-eval/SKILL.md`](../../skills/ddtest-onboarding-eval/SKILL.md). diff --git a/internal/onboard/ci_provider.go b/internal/onboard/ci_provider.go index 900ee0c5f..d86c69a2f 100644 --- a/internal/onboard/ci_provider.go +++ b/internal/onboard/ci_provider.go @@ -5,17 +5,6 @@ package onboard -import ( - "fmt" - "maps" - "slices" - "strings" - "sync/atomic" - - "github.com/kballard/go-shellquote" - "mvdan.cc/sh/v3/syntax" -) - // ciProvider is everything onboarding needs from one CI provider. A provider // parses its own configuration into the shared workflow model (jobs, steps, // commands, environment, matrix and conditions); what to run and how to set up @@ -41,10 +30,6 @@ type ciProvider interface { configurationFiles(root string) ([]string, error) // readWorkflows parses the provider's configuration into the shared model. readWorkflows(root string) ([]ciWorkflow, error) - // findTestWorkflows lists the configuration files whose jobs run - // framework's tests, and the subset that already set up Test Optimization - // for language through the provider's setup mechanism. - findTestWorkflows(root, language, framework string) (workflows, configured []string, err error) // environment holds the variables the provider sets in every job. environment() map[string]string @@ -86,797 +71,3 @@ type ciProvider interface { // impact lists consequences a person should review. planSetupEdits(root string, workflows []ciWorkflow, targets []ciSetupTarget, request ciSetupRequest) (edits []setupEdit, impact []string, err error) } - -// ciSetup is the Datadog Test Optimization setup one CI step runs, described -// in the setup action's terms. Each provider recognizes its own mechanism -// (see ciProvider.setupStep); shared code never inspects the mechanism. -type ciSetup struct { - // Reference names what the step runs, such as the action and its ref. - Reference string - // Pinned reports a reference that selects a release. - Pinned bool - // Supported reports a release ddtest validates, with its integrity check - // where the provider has one. - Supported bool - // DefaultLatest reports that an unset tracer version input installs the - // latest release; otherwise the mechanism's metadata defines the default. - DefaultLatest bool - // Problem is why the setup cannot work where it runs, as the provider - // found while reading it; setupProblem reports it first. - Problem string - // shaped reports a setup step in a form the provider recognizes entirely. - shaped bool - // fallback reports ddtest's setup without Bash or curl (see - // fallbackSetupAt), which every provider validates the same way. - fallback bool - // after are shell statements that run only with the setup, inside its - // guard. - after []*syntax.Stmt - // before are shell statements a setup snippet runs before itself, - // whether or not it installs anything (see snippetSetupAt). - before []string - // shellLocal reports a setup whose exports reach only the later - // commands of the shell it runs in, such as a setup snippet in a - // Jenkinsfile sh step: a step of its job in another shell does not have - // them (see ciShellOrigin). - shellLocal bool - // Inputs are the installer inputs keyed by the setup action's input - // names: languages, api_key, site and -tracer-version. Values - // are verbatim; unresolved references stay as written. - Inputs map[string]string - // Floating reports a mechanism that cannot pin the tracer: it installs - // the latest release when the build runs, unless the project's own - // dependencies already pin one. Each language decides which applies and - // reports the floating release as a caveat. - Floating bool - // WorkspaceRoot reports a mechanism that configures the tracer from the - // checkout root with the build agent's own toolchain, so a tracer that - // joins the project's dependencies must join the root ones. - WorkspaceRoot bool - // Confirmations are the facts (--ci-confirm keys, see ciProviderFacts) - // the user must state before ddtest certifies this setup. - Confirmations []string - // Caveats are what ddtest could not check about the mechanism, for - // final_response. - Caveats []string - // Beside names another Datadog setup mechanism the build enables for - // the same language in other jobs that share its workspace, such as the - // Datadog Jenkins plugin beside a container stage's setup snippet; - // languages whose tracer joins the project's dependencies check that it - // cannot change the release this setup installs. - Beside string - // Exports are the loader variables the mechanism sets for the steps - // after it besides the action's DD_TRACE_PACKAGE and DD_TRACE_ESM_IMPORT, - // such as the Datadog Jenkins plugin's DD_TRACE_PATH. A test step's - // NODE_OPTIONS can trust only the variables its own setup exports. - Exports []string -} - -// ciSetupRequest is the setup one language asks a provider for: the -// provider supplies the step, the language its own values. -type ciSetupRequest struct { - // Language is the installer's token for the language (languages input). - Language string - // Site is the Datadog site the setup reports to. - Site string - // TracerInput names the exact tracer release input, such as - // js-tracer-version, and TracerVersion its value. - TracerInput, TracerVersion string - // After are shell commands that must follow the setup and precede the - // tests in the same job, such as an ESM loader registration. - After []string - // Notes are language rules the instructions add after the step. - Notes []string - // Service is the service name a setup that names one reports under: the - // project's own name, empty when the language reads none. - Service string - // Framework is the framework whose tests the setup is for. - Framework string - // Jobs are the jobs the setup is for, when the provider sets its jobs up - // with several mechanisms (see runtimeJob.SetupProvider); empty for all. - Jobs []string - // Shared are the job definitions of these jobs whose other invocations - // run no tests (see circleSharedJobs): the setup is enabled only for the - // invocations that run the tests. - Shared []ciSharedJob -} - -// ciSetupTerms are how a provider without the shared workflow syntax names -// its setup in the languages' setup notes, which refer to the setup's place -// in a job and to how a later command keeps a variable it exports. -type ciSetupTerms struct { - // Installer names the setup mechanism after "the", such as "script". - Installer string - // Entry names the setup's place in a job, such as "the entry", and - // SetupEntry the same place where the setup must be named, such as "the - // setup entry". - Entry, SetupEntry string - // Later names what the setup's exports reach, such as "later commands of - // the job". - Later string - // Variables names where a job declares its variables, such as "the job's - // variables". - Variables string - // Persist describes, in a sentence, running a shell command after the - // setup so that later test commands see what it exports; PersistEntry - // writes it as an entry of the job's configuration (a YAML list item). - Persist, PersistEntry func(command string) string - // MatrixExclusion says how matrix entries leave a job, such as "with - // rules (when: never on their matrix variables)". - MatrixExclusion string - // SharedMatrix names a matrix whose entries share one job's commands, - // such as "a parallel:matrix shares the script". - SharedMatrix string - // ImageGuard writes an entry that runs command only where the job's - // image, written as template, is image; empty when the template selects - // no image through a variable or parameter. - ImageGuard func(template, image, command string) string - // Beside names the provider's other setup mechanism that the same build - // runs for the language in other jobs of its workspace (see - // runtimeJob.SetupProvider), such as the Datadog Jenkins plugin beside - // container stages' setup snippet; empty when there is none. - Beside string - // SecretStore names where the provider keeps the DD_API_KEY a job reads - // without writing it, such as "a masked GitLab CI/CD variable", for - // setups written by hand. - SecretStore string - // KeyBinding reports a provider that gives a job the stored key only - // where its configuration binds it (a Jenkins credential), not merely - // because the secret store holds it: a hand-written setup without - // DD_API_KEY in its environment then reports nothing. - KeyBinding bool - // Fallback reports the setup without Bash or curl (see imageFallback), - // which installs the tracer with the job's own tools: languages describe - // what it exports. - Fallback bool - // Plugin reports a setup that is a CI server plugin, enabled in the - // configuration, that installs the tracers on the build agent and sets - // their variables for every step (the Datadog Jenkins plugin), instead of - // an installer the job runs: languages describe that mechanism. - Plugin bool -} - -// ciComponent is a reusable CI component a language knows, such as a -// runtime or dependency installer published for one provider, modeled in the -// provider's own syntax so the provider reads it like the repository's -// configuration. -type ciComponent struct { - // Name is the component's registry name, such as an orb's - // namespace/name. - Name string - // Majors are the major versions the model describes. - Majors []string - // Definition is the model: for CircleCI an inline orb's commands, jobs - // and executors. - Definition string -} - -// ciSetupTarget is one test step that needs the setup before it. -type ciSetupTarget struct { - Workflow, Job string - // Step indexes the job's steps. - Step int - // After overrides the request's commands for this step. - After []string -} - -// datadogSetup recognizes the Datadog setup a step runs, whichever provider -// read it. -func datadogSetup(step runtimeStep) (ciSetup, bool) { - if step.Setup != nil { - return *step.Setup, true - } - for _, provider := range ciProviders { - if setup, ok := provider.setupStep(step); ok { - return setup, true - } - } - return ciSetup{}, false -} - -// setupProviderFor is the provider whose setup mechanism job uses: its -// own, unless the job runs another of its mechanisms (see -// runtimeJob.SetupProvider), whose names, inputs and instructions then -// apply. -func setupProviderFor(provider ciProvider, job runtimeJob) ciProvider { - if job.SetupProvider != nil { - return job.SetupProvider - } - return provider -} - -// jobSetups are the jobs that set Test Optimization up with one of a -// provider's mechanisms. -type jobSetups struct { - provider ciProvider - jobs []string -} - -// testJobSetups groups the jobs that run framework's tests by the setup -// mechanism each uses (see setupProviderFor), the provider's own first; a -// provider with one mechanism, or no such job, gives that one alone. -func testJobSetups(root string, provider ciProvider, language, framework string) []jobSetups { - groups := []jobSetups{{provider: provider}} - workflows, err := readProviderWorkflows(provider, root) - if err != nil { - return groups - } - for _, workflow := range workflows { - for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { - job := workflow.Jobs[name] - if disabledCondition(job.If) || !slices.ContainsFunc(job.Steps, func(step runtimeStep) bool { - return resolveTestStep(root, workflow, job, step, language, framework).Matched - }) { - continue - } - mechanism := setupProviderFor(provider, job) - i := slices.IndexFunc(groups, func(group jobSetups) bool { return group.provider == mechanism }) - if i < 0 { - groups, i = append(groups, jobSetups{provider: mechanism}), len(groups) - } - groups[i].jobs = append(groups[i].jobs, workflow.Path+" / "+name) - } - } - if len(groups) > 1 && len(groups[0].jobs) == 0 { - groups = groups[1:] - } - return groups -} - -// setupProviderNames names the setup mechanisms of groups. -func setupProviderNames(groups []jobSetups) string { - names := make([]string, 0, len(groups)) - for _, group := range groups { - if name := group.provider.setupName(); !slices.Contains(names, name) { - names = append(names, name) - } - } - return strings.Join(names, " or the ") -} - -// setupSelectsLanguage reports whether setup installs the tracer of the -// language named by its languages input, after resolving row's matrix values. -func setupSelectsLanguage(setup ciSetup, row map[string]any, language string) (bool, error) { - languages, err := runtimeValue(setup.Inputs["languages"], row) - if err != nil { - return false, err - } - return languages == "all" || slices.Contains(strings.Fields(languages), language), nil -} - -// CIProviderSetupName names the selected CI provider's setup mechanism when -// it is not written in the shared workflow syntax; empty otherwise, where -// languages name the setup action themselves. -func CIProviderSetupName() string { - if provider := activeOrDefaultCIProvider(); !provider.sharedSyntax() { - return provider.setupName() - } - return "" -} - -// CIProviderTracerSetting is how the selected CI provider's setup pins -// the language's tracer release, such as js-tracer-version: 5.133.0, or -// DD_SET_TRACER_VERSION_JS="5.133.0" in the Buildkite setup snippet. -func CIProviderTracerSetting(language, version string) string { - provider := activeOrDefaultCIProvider() - input := provider.setupInput(language + "-tracer-version") - switch provider.name() { - case buildkiteProvider: - return input + `="` + version + `"` - case jenkinsProvider: - return "release " + version + " where the CI setup can pin it (the " + input + ")" - } - return input + ": " + version -} - -// CIProviderSiteSetting names site in place of the example Datadog site the -// manual setup instructions show, which they render for the default site: -// empty for that default, which needs no substitution. -func CIProviderSiteSetting(site string) string { - if site == "" || site == defaultDatadogSite { - return "" - } - return "the Datadog site " + site + " in place of the example site " + defaultDatadogSite -} - -// CIProviderAPIKeyAction is the person's step that stores the Datadog API key -// for the selected CI provider. -func CIProviderAPIKeyAction(skipped bool) string { - return activeOrDefaultCIProvider().apiKeyAction(skipped) -} - -// ciAPIKeyActions is a provider whose jobs set Test Optimization up with -// several mechanisms (see runtimeJob.SetupProvider), each receiving the -// API key its own way. -type ciAPIKeyActions interface { - // apiKeyActions are the person's steps for the mechanisms named by - // references (ciSetup.Reference of the validated jobs' setups). - apiKeyActions(references []string, skipped bool) string -} - -// CIProviderAPIKeyActionFor is CIProviderAPIKeyAction for the setups of -// check's jobs, on a provider whose jobs use several mechanisms. -func CIProviderAPIKeyActionFor(check *RuntimeCheck, skipped bool) string { - provider := activeOrDefaultCIProvider() - several, ok := provider.(ciAPIKeyActions) - if !ok && check != nil { - // Jobs on the setup without Bash or curl skip it without the key. - fallback := slices.ContainsFunc(check.Jobs, func(job RuntimeFinding) bool { return job.Action == fallbackSetupName }) - others := slices.ContainsFunc(check.Jobs, func(job RuntimeFinding) bool { return job.Action != "" && job.Action != fallbackSetupName }) - switch { - case fallback && others: - return provider.apiKeyAction(skipped) + " " + newImageFallback(provider, "", "").apiKeyAction(skipped) - case fallback: - return newImageFallback(provider, "", "").apiKeyAction(skipped) - } - } - if !ok || check == nil { - return provider.apiKeyAction(skipped) - } - var references []string - for _, job := range check.Jobs { - if job.Action != "" { - references = append(references, job.Action) - } - } - return several.apiKeyActions(references, skipped) -} - -// CIProviderChoice is a user decision: several CI providers contain tests and -// ddtest configures one at a time. No command is suggested until it is made. -type CIProviderChoice struct{ Providers []string } - -func (e *CIProviderChoice) Error() string { - return "Several CI providers contain test commands: " + strings.Join(e.Providers, ", ") + ". Ask the user which provider to install Test Optimization on, then rerun with --ci-provider ." -} - -// CIProviderUnsupportedError reports a language without CI setup support on -// the selected provider, or (Reason) a repository whose every test job is -// outside what the provider's setup mechanism instruments. ddtest -// prescribes nothing for that combination. -type CIProviderUnsupportedError struct { - Language, Framework, Provider string - // Reason names the jobs the provider's setup cannot instrument, and - // why; Setup names that setup. - Reason, Setup string - // HandWritten names the jobs that set up Test Optimization by hand in - // a way ddtest recognizes but does not validate for Language yet (see - // handWrittenSetup.Unvalidated), each with its setup. - HandWritten []string -} - -func (e *CIProviderUnsupportedError) Error() string { - if len(e.HandWritten) > 0 { - return fmt.Sprintf("ddtest cannot validate %s CI for %s on %s in this repository: ddtest does not validate hand-written %s setups yet, and these CI jobs set up Test Optimization by hand: %s ddtest neither certifies those setups nor replaces them with the %s. Keep them as they are, and do not add the %s to those jobs, remove their setup or change how they report; what a setup lacks is for the user to decide. ddtest will not prescribe CI edits or validate this CI configuration.", e.Framework, e.Language, e.Provider, handWrittenLanguageName(e.Language), strings.TrimSuffix(strings.Join(e.HandWritten, "; "), ".")+".", e.Setup, e.Setup) - } - if e.Reason != "" { - return fmt.Sprintf("ddtest cannot set up %s CI for %s on %s in this repository: every CI job that runs the tests is outside what the %s instruments. %s The user's options are to run those tests where the %s instruments them, or %s. ddtest will not prescribe CI edits or validate this CI configuration as it is.", e.Framework, e.Language, e.Provider, e.Setup, e.Reason, e.Setup, handWrittenSetupOption(e.Language, e.Setup)) - } - return fmt.Sprintf("ddtest does not support %s CI setup for %s on %s; it will not prescribe CI edits or validate this CI configuration. Rerun with --ci-provider to choose another provider.", e.Framework, e.Language, e.Provider) -} - -// activeProvider is the provider selected for this command; empty selects one -// automatically. -var activeProvider atomic.Value - -// UseCIProvider selects the CI provider onboarding and testdrive read for the -// rest of the command, normally the result of SelectCIProvider. Empty selects -// it automatically from the repository again. -func UseCIProvider(name string) { activeProvider.Store(name) } - -// ActiveCIProvider names the provider selected with UseCIProvider, or empty. -func ActiveCIProvider() string { - name, _ := activeProvider.Load().(string) - return name -} - -// CIProviderNames lists the supported --ci-provider values. -func CIProviderNames() []string { - names := make([]string, 0, len(ciProviders)) - for _, provider := range ciProviders { - names = append(names, provider.name()) - } - return names -} - -// UnknownCIProviderError rejects a --ci-provider value no provider registers. -func UnknownCIProviderError(name string) error { - return fmt.Errorf("unsupported --ci-provider %q; supported CI providers: %s", name, strings.Join(CIProviderNames(), ", ")) -} - -func ciProviderNamed(name string) ciProvider { - for _, provider := range ciProviders { - if provider.name() == name { - return provider - } - } - return nil -} - -// defaultCIProvider applies when a repository has no CI configuration: the -// first registered provider, as onboarding behaved before providers were -// selectable. -func defaultCIProvider() ciProvider { return ciProviders[0] } - -// SelectCIProvider resolves the CI provider to configure. A requested provider -// is used as given. Otherwise the only provider with configuration files is -// used without parsing them; with several, the ones whose configuration runs -// tests or leaves steps unresolved are candidates, and more than one candidate -// returns a *CIProviderChoice. A configuration that cannot be parsed never -// fails selection: that provider is not a candidate, and its files stay -// visible as review notes of the selected provider's scope. Empty means no -// provider has configuration files. -func SelectCIProvider(root, requested string) (string, error) { - if requested != "" { - if ciProviderNamed(requested) == nil { - return "", UnknownCIProviderError(requested) - } - return requested, nil - } - var configured []ciProvider - for _, provider := range ciProviders { - // A listing error is reported when that provider is read. - if files, err := provider.configurationFiles(root); err != nil || len(files) > 0 { - configured = append(configured, provider) - } - } - switch len(configured) { - case 0: - return "", nil - case 1: - return configured[0].name(), nil - } - var candidates []string - for _, provider := range configured { - workflows, err := provider.readWorkflows(root) - if err == nil && slices.ContainsFunc(workflows, func(workflow ciWorkflow) bool { return workflowHasTests(root, workflow) }) { - candidates = append(candidates, provider.name()) - } - } - switch len(candidates) { - case 0: - return configured[0].name(), nil - case 1: - return candidates[0], nil - } - return "", &CIProviderChoice{Providers: candidates} -} - -// currentCIProvider is the provider selected with UseCIProvider, otherwise -// the one SelectCIProvider resolves, otherwise the default. -func currentCIProvider(root string) (ciProvider, error) { - name := ActiveCIProvider() - if name == "" { - var err error - if name, err = SelectCIProvider(root, ""); err != nil { - return nil, err - } - } - if name == "" { - return defaultCIProvider(), nil - } - if provider := ciProviderNamed(name); provider != nil { - return provider, nil - } - return nil, UnknownCIProviderError(name) -} - -// displayCIText writes the shared model's checkout expression, which every -// provider's values use, in the selected provider's own syntax, for output. -func displayCIText(text string) string { - reference := activeOrDefaultCIProvider().checkoutReference() - return strings.NewReplacer(ciProjectDirectory, reference, "${{github.workspace}}", reference).Replace(text) -} - -// displayCIError is err with its message in the selected provider's syntax -// (see displayCIText); it still wraps err. -func displayCIError(err error) error { - if err == nil || !strings.Contains(err.Error(), "github.workspace") || activeOrDefaultCIProvider().checkoutReference() == ciProjectDirectory { - return err - } - return displayedCIError{err} -} - -type displayedCIError struct{ err error } - -func (e displayedCIError) Error() string { return displayCIText(e.err.Error()) } -func (e displayedCIError) Unwrap() error { return e.err } - -// activeOrDefaultCIProvider is the selected provider without consulting the -// repository: the default until UseCIProvider selects another. -func activeOrDefaultCIProvider() ciProvider { - if provider := ciProviderNamed(ActiveCIProvider()); provider != nil { - return provider - } - return defaultCIProvider() -} - -// readWorkflows reads the selected provider's CI configuration. -func readWorkflows(root string) ([]ciWorkflow, error) { - provider, err := currentCIProvider(root) - if err != nil { - return nil, err - } - return readProviderWorkflows(provider, root) -} - -// WithCIProviderDefaults adds the variables the selected CI provider sets in -// every job, without overriding the configuration's own. -func WithCIProviderDefaults(env map[string]string) map[string]string { - return mergeEnvironment(activeOrDefaultCIProvider().environment(), env) -} - -// CIProviderConfigurationFiles lists the selected CI provider's configuration -// files in root without parsing them. -func CIProviderConfigurationFiles(root string) ([]string, error) { - return activeOrDefaultCIProvider().configurationFiles(root) -} - -// CIConfigurationFiles lists every repository file the selected CI -// provider's configuration is read from, included files too. A -// configuration that cannot be read lists its configuration files only. -func CIConfigurationFiles(root string) []string { - provider := activeOrDefaultCIProvider() - files, _ := provider.configurationFiles(root) - workflows, err := provider.readWorkflows(root) - if err != nil { - return files - } - for _, workflow := range workflows { - for _, file := range append([]string{workflow.Path}, workflow.Files...) { - if !slices.Contains(files, file) { - files = append(files, file) - } - } - } - return files -} - -// ciProviderFollowUpFlag returns the --ci-provider argument human follow-up -// commands need: only when several providers have configuration in root, so -// a rerun would ask again. -func ciProviderFollowUpFlag(root string) string { - return ciProviderFlag(root) + ciStatementsFollowUp() -} - -// ciStatementsFollowUp repeats the user's CI statements in a human follow-up -// command, which reports are bound to. -func ciStatementsFollowUp() string { - if arguments := CIStatementArguments(); len(arguments) > 0 { - return " " + shellquote.Join(arguments...) - } - return "" -} - -func ciProviderFlag(root string) string { - selected := ActiveCIProvider() - if selected == "" { - return "" - } - configured := 0 - for _, provider := range ciProviders { - if files, err := provider.configurationFiles(root); err != nil || len(files) > 0 { - configured++ - } - } - if configured < 2 { - return "" - } - return " --ci-provider " + selected -} - -// CIProviderDisplayName names the selected CI provider in output. -func CIProviderDisplayName() string { return activeOrDefaultCIProvider().displayName() } - -// languageSupportsCIProvider reports whether language sets up Test -// Optimization on provider. Unregistered languages support none. -func languageSupportsCIProvider(language string, provider ciProvider) bool { - registered := discoveryLanguageNamed(language) - return registered != nil && registered.supportsCIProvider(provider.name()) -} - -// CheckCIProviderSupport returns a *CIProviderUnsupportedError when the -// selected CI provider has no setup support for language's framework. -func CheckCIProviderSupport(root, language, framework string) error { - provider, err := currentCIProvider(root) - if err != nil { - return err - } - // Without the provider's configuration there is no CI to consult or set up. - if files, err := provider.configurationFiles(root); err == nil && len(files) == 0 { - return nil - } - return providerSupportError(root, provider, language, framework) -} - -// providerSupportError is CheckCIProviderSupport's result once the -// provider's configuration exists. A hand-written setup ddtest recognizes -// but does not validate for language comes first, whatever the provider -// supports, so it is never replaced or certified. -func providerSupportError(root string, provider ciProvider, language, framework string) error { - if jobs := unvalidatedHandWrittenJobs(root, provider, language, framework); len(jobs) > 0 { - return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName(), Setup: provider.setupName(), HandWritten: jobs} - } - if !languageSupportsCIProvider(language, provider) { - return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName()} - } - if reason := unsupportedTestJobs(root, provider, language, framework); reason != "" { - return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName(), Reason: reason, Setup: provider.setupName()} - } - return nil -} - -// unsupportedTestJobs explains, when every CI job that runs framework's -// tests is one the provider's setup mechanism cannot instrument (see -// runtimeJob.SetupUnsupported), why; empty when any can, or none runs them. -func unsupportedTestJobs(root string, provider ciProvider, language, framework string) string { - workflows, err := readProviderWorkflows(provider, root) - if err != nil { - return "" - } - var reasons []string - for _, workflow := range workflows { - for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { - job := workflow.Jobs[name] - if disabledCondition(job.If) { - continue - } - tests := slices.ContainsFunc(job.Steps, func(step runtimeStep) bool { - resolution := resolveTestStep(root, workflow, job, step, language, framework) - return !resolution.Review && (resolution.Matched || resolution.Reason != "" && step.ResolutionError == "" && stepMayStartTests(root, workflow, job, step, framework)) - }) - if !tests { - continue - } - // A job that sets up the tracer by hand needs no provider setup. - if job.SetupUnsupported == "" || firstHandWrittenSetup(root, workflow, job, language, framework, "") != nil { - return "" - } - reasons = append(reasons, workflow.Path+" / "+name+": "+job.SetupUnsupported+".") - } - } - return strings.Join(reasons, " ") -} - -// workflowHasTests reports whether any active step of workflow may run tests -// of a registered language, or cannot be resolved well enough to tell. -func workflowHasTests(root string, workflow ciWorkflow) bool { - for _, job := range workflow.Jobs { - if disabledCondition(job.If) { - continue - } - if _, _, err := jobMatrix(job); err != nil { - return true // An unresolved matrix can run tests. - } - for _, step := range job.Steps { - if disabledCondition(step.If) { - continue - } - if step.Run == "" && step.ResolutionError != "" { - return true - } - for _, language := range discoveryLanguages { - for _, framework := range language.frameworks() { - resolution := resolveTestStep(root, workflow, job, step, language.name(), framework) - if resolution.Matched || (!resolution.Review && resolution.Reason != "" && stepMayStartTests(root, workflow, job, step, framework)) || namesFramework(step.Run, framework, true) { - return true - } - } - } - } - } - return false -} - -// disabledCondition reports whether condition can never pass, such as false or -// ${{ false }}; anything unresolved may still run. -func disabledCondition(condition string) bool { - if strings.TrimSpace(condition) == "" { - return false - } - active, err := possibleJobCondition(condition, nil) - return err == nil && !active -} - -// ProviderChoiceResult asks the user to choose the CI provider to configure. -func ProviderChoiceResult(choice *CIProviderChoice) AgentResult { - return AgentResult{Status: "needs_input", Phase: "discovery", Code: "ci_provider_required", Reason: choice.Error(), RequiredInput: choice.Providers, NextAction: "ask_user", Instruction: "Ask the user which CI provider should receive Test Optimization. Do not choose one or install on every provider. After their answer, rerun the original command with --ci-provider ."} -} - -// ProviderUnsupportedResult reports a language/provider combination ddtest -// does not set up. -func ProviderUnsupportedResult(err *CIProviderUnsupportedError) AgentResult { - result := AgentResult{Status: "unsupported", Phase: "onboarding", Code: "ci_provider_unsupported", Reason: err.Error(), NextAction: "report_result"} - switch { - case len(err.HandWritten) > 0: - result.Code = "ci_setup_unsupported" - result.Instruction = "Report this result to the user with its reason and stop. Keep the hand-written setups as they are: do not add the " + err.Setup + ", edit CI, add or change a tracer, or remove a setup; those are the user's decisions, and ddtest cannot validate hand-written " + handWrittenLanguageName(err.Language) + " setups yet." - case err.Reason != "": - result.Code = "ci_setup_unsupported" - result.Instruction = "Report this result to the user with its reason and stop. Do not move tests between agents, edit CI, add a tracer or set up Test Optimization by hand: those are the user's decisions." - if validatesHandWrittenSetups(err.Language) { - result.Instruction += " Once the user sets it up by hand, rerun onboarding, which checks that setup." - } else { - result.Instruction += " ddtest does not validate hand-written " + handWrittenLanguageName(err.Language) + " setups yet." - } - } - return result -} - -// CIProviderEditsSetup reports whether onboard --apply writes the selected -// CI provider's setup; otherwise every language gets instructions. -func CIProviderEditsSetup() bool { return activeOrDefaultCIProvider().editsSetup() } - -// providerAutomaticSetup reports whether onboard --apply writes framework's -// setup on provider in root, reporting to site (empty: any): the provider -// edits its configuration and the language writes that framework's setup -// there. A plugin setup applies only where the provider can make the edit -// and leave it complete for site; otherwise the instructions do. -func providerAutomaticSetup(root string, provider ciProvider, language discoveryLanguage, framework, site string) bool { - return provider.editsSetup() && language.automaticProviderSetup(framework) && (!provider.setupTerms().Plugin || pluginSetupApplies(root, framework, site)) -} - -// testWorkflowsWithSetup lists the workflows whose jobs run framework's tests -// and those where an active setup selecting language precedes every test step. -func testWorkflowsWithSetup(root string, workflows []ciWorkflow, language, framework string) (found, configured []string) { - token := setupActionLanguage(language) - for _, workflow := range workflows { - tests, complete := false, true - for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { - job := workflow.Jobs[name] - if disabledCondition(job.If) { - continue - } - setup := false - for i, step := range job.Steps { - if current, ok := datadogSetup(step); ok { - selected, err := setupSelectsLanguage(current, nil, token) - setup = setup || err == nil && selected - } - resolution := resolveTestStep(root, workflow, job, step, language, framework) - if resolution.Review || !resolution.Matched && (resolution.Reason == "" || !stepMayStartTests(root, workflow, job, step, framework)) { - continue - } - tests = true - // A complete setup the job writes by hand counts as its own. - handWritten := !setup && resolution.Matched && resolution.Reason == "" && func() bool { - current := handWrittenCISetup(root, workflow, job, i, nil, language, framework, "") - return current != nil && current.complete() - }() - complete = complete && (setup || handWritten) && resolution.Reason == "" - } - } - if tests { - found = append(found, workflow.Path) - if complete { - configured = append(configured, workflow.Path) - } - } - } - return found, configured -} - -// ciConfirmationProblem lists the provider facts the user has not -// confirmed (--ci-confirm) that this entry's setup depends on, and how to -// confirm them; empty when every one is confirmed. -func ciConfirmationProblem(provider ciProvider, setup ciSetup) string { - var missing []ciFactDefinition - for _, definition := range ciProviderFacts[provider.name()] { - if _, stated := statedFact(definition.Key); slices.Contains(setup.Confirmations, definition.Key) && !stated { - missing = append(missing, definition) - } - } - if len(missing) == 0 { - return "" - } - var questions, flags []string - for _, fact := range missing { - flag := fact.Key - if fact.Values != "" { - flag += "=" + fact.Values - } - questions = append(questions, strings.TrimSpace(fact.Key+": "+fact.Question+" "+fact.Remedy)) - flags = append(flags, "--ci-confirm "+flag) - } - return "The " + provider.setupName() + " setup depends on facts no CI configuration shows, so ddtest cannot certify it until the user confirms them. Ask the user; never assume or check them from this machine. " + strings.Join(questions, " ") + " Then rerun testdrive adding " + strings.Join(flags, " ") + " with the values they confirmed." -} diff --git a/internal/onboard/ci_provider_buildkite.go b/internal/onboard/ci_provider_buildkite.go index 9e5cef85f..87f127e64 100644 --- a/internal/onboard/ci_provider_buildkite.go +++ b/internal/onboard/ci_provider_buildkite.go @@ -136,15 +136,6 @@ func (b buildkite) readWorkflows(root string) ([]ciWorkflow, error) { return workflows, nil } -func (b buildkite) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - workflows, err := b.readWorkflows(root) - if err != nil { - return nil, nil, err - } - found, configured := testWorkflowsWithSetup(root, workflows, language, framework) - return found, configured, nil -} - func (buildkite) setupStep(step runtimeStep) (ciSetup, bool) { if step.Setup == nil { return ciSetup{}, false diff --git a/internal/onboard/ci_provider_circleci.go b/internal/onboard/ci_provider_circleci.go index 0d25ea215..e33c159e5 100644 --- a/internal/onboard/ci_provider_circleci.go +++ b/internal/onboard/ci_provider_circleci.go @@ -82,15 +82,6 @@ func (c circleCI) readWorkflows(root string) ([]ciWorkflow, error) { return []ciWorkflow{config.workflow()}, nil } -func (c circleCI) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - workflows, err := c.readWorkflows(root) - if err != nil { - return nil, nil, err - } - found, configured := testWorkflowsWithSetup(root, workflows, language, framework) - return found, configured, nil -} - func (circleCI) setupStep(step runtimeStep) (ciSetup, bool) { if step.Setup == nil { return ciSetup{}, false diff --git a/internal/onboard/ci_provider_github.go b/internal/onboard/ci_provider_github.go index 8681abc07..074f5721c 100644 --- a/internal/onboard/ci_provider_github.go +++ b/internal/onboard/ci_provider_github.go @@ -10,9 +10,10 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" - "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/testcommand" "go.yaml.in/yaml/v3" ) @@ -61,6 +62,22 @@ func (g gitHubActions) configurationFiles(root string) ([]string, error) { } func (g gitHubActions) readWorkflows(root string) ([]ciWorkflow, error) { + workflows, err := g.readWorkflowFiles(root) + if err != nil { + return nil, err + } + for _, workflow := range workflows { + for jobName, job := range workflow.Jobs { + job.Steps = expandCompositeSteps(root, workflow.Path, job.Steps) + workflow.Jobs[jobName] = job + } + } + return workflows, nil +} + +// readWorkflowFiles is the single GitHub workflow loader. Manual instructions +// inspect literal steps; discovery expands composite actions after loading. +func (g gitHubActions) readWorkflowFiles(root string) ([]ciWorkflow, error) { names, err := g.workflowEntries(root) if err != nil { return nil, err @@ -78,17 +95,64 @@ func (g gitHubActions) readWorkflows(root string) ([]ciWorkflow, error) { return nil, fmt.Errorf("parse %s: %w", path, err) } workflow.Path = ".github/workflows/" + name - for jobName, job := range workflow.Jobs { - job.Steps = expandCompositeSteps(root, workflow.Path, job.Steps) - workflow.Jobs[jobName] = job - } workflows = append(workflows, workflow) } return workflows, nil } -func (gitHubActions) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - return environment.NewGitHub().FindTestWorkflows(root, language, framework) +func (g gitHubActions) findTestWorkflows(root, language, framework string) ([]string, []string, error) { + workflows, err := g.readWorkflowFiles(root) + if err != nil { + return nil, nil, err + } + var found, configured []string + for _, workflow := range workflows { + tests, setup := inspectLiteralTestJobs(root, workflow, language, framework) + if tests { + found = append(found, workflow.Path) + if setup { + configured = append(configured, workflow.Path) + } + } + } + return found, configured, nil +} + +// inspectLiteralTestJobs preserves the conservative action and command checks +// used by manual onboarding. Full validation uses resolved commands instead. +func inspectLiteralTestJobs(root string, workflow ciWorkflow, language, name string) (foundTests, configured bool) { + actionLanguage := setupActionLanguage(language) + for _, job := range workflow.Jobs { + jobConfigured := false + for _, step := range job.Steps { + condition := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(step.If), "${{"), "}}")) + if condition == "false" { + continue + } + directory := step.WorkingDirectory + if directory == "" { + directory = job.Defaults.Run.WorkingDirectory + } + if directory == "" { + directory = workflow.Defaults.Run.WorkingDirectory + } + if !filepath.IsAbs(directory) { + directory = filepath.Join(root, directory) + } + if testcommand.Matches(directory, step.Run, language, name) { + foundTests = true + if !jobConfigured { + return true, false + } + } + action, _, hasVersion := strings.Cut(strings.ToLower(step.Uses), "@") + if action == githubAction && hasVersion && (condition == "" || condition == "true" || condition == "success()") && !strings.Contains(step.With["languages"], "${{") { + languages := strings.Fields(step.With["languages"]) + jobConfigured = jobConfigured || step.With["languages"] == "all" || slices.Contains(languages, actionLanguage) + } + } + } + return foundTests, foundTests } func (gitHubActions) editsSetup() bool { return true } diff --git a/internal/environment/github_workflows_test.go b/internal/onboard/ci_provider_github_test.go similarity index 76% rename from internal/environment/github_workflows_test.go rename to internal/onboard/ci_provider_github_test.go index fc4bbe0b5..a9dff8d8f 100644 --- a/internal/environment/github_workflows_test.go +++ b/internal/onboard/ci_provider_github_test.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026 Datadog, Inc. -package environment +package onboard import ( "fmt" @@ -54,7 +54,7 @@ jobs: `, tc.workflow, tc.job, tc.step) require.NoError(t, os.MkdirAll(filepath.Join(root, ".github/workflows"), 0755)) require.NoError(t, os.WriteFile(filepath.Join(root, ".github/workflows/test.yml"), []byte(workflow), 0644)) - workflows, configured, err := NewGitHub().FindTestWorkflows(root, "javascript", "jest") + workflows, configured, err := (gitHubActions{}).findTestWorkflows(root, "javascript", "jest") require.NoError(t, err) require.Equal(t, tc.found, len(workflows) == 1) require.Equal(t, tc.found, len(configured) == 1) @@ -98,7 +98,7 @@ func TestGitHubFindTestWorkflowsCommandKinds(t *testing.T) { path := filepath.Join(root, ".github/workflows/test.yml") workflow := fmt.Sprintf("jobs:\n tests:\n steps:\n - run: %s\n", tc.command) require.NoError(t, os.WriteFile(path, []byte(workflow), 0644)) - workflows, configured, err := NewGitHub().FindTestWorkflows(root, tc.language, tc.framework) + workflows, configured, err := (gitHubActions{}).findTestWorkflows(root, tc.language, tc.framework) require.NoError(t, err) require.Equal(t, tc.tests, len(workflows) == 1) require.Empty(t, configured) @@ -107,7 +107,7 @@ func TestGitHubFindTestWorkflowsCommandKinds(t *testing.T) { // that run after the Datadog action. workflow += " - uses: datadog/test-visibility-github-action@v3\n with: {languages: all}\n - run: " + tc.framework + " run\n" require.NoError(t, os.WriteFile(path, []byte(workflow), 0644)) - workflows, configured, err = NewGitHub().FindTestWorkflows(root, tc.language, tc.framework) + workflows, configured, err = (gitHubActions{}).findTestWorkflows(root, tc.language, tc.framework) require.NoError(t, err) require.Len(t, workflows, 1) require.Equal(t, !tc.tests, len(configured) == 1) @@ -133,10 +133,41 @@ func TestGitHubFindTestWorkflowsRecognizesMavenTestPhases(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(root, ".github/workflows"), 0755)) workflow := fmt.Sprintf("jobs:\n tests:\n steps:\n - uses: datadog/test-visibility-github-action@v3\n with: {languages: java}\n - run: %s\n", tc.command) require.NoError(t, os.WriteFile(filepath.Join(root, ".github/workflows/test.yml"), []byte(workflow), 0644)) - workflows, configured, err := NewGitHub().FindTestWorkflows(root, "java", "junit5") + workflows, configured, err := (gitHubActions{}).findTestWorkflows(root, "java", "junit5") require.NoError(t, err) require.Equal(t, tc.tests, len(workflows) == 1) require.Equal(t, tc.tests, len(configured) == 1, "the action selects java") }) } } + +func TestGitHubWorkflowLoadingIsShared(t *testing.T) { + root := t.TempDir() + provider := gitHubActions{} + files, err := provider.configurationFiles(root) + require.NoError(t, err) + require.Empty(t, files) + for _, name := range []string{"z.yml", "a.yaml", "notes.txt"} { + writeFile(t, root, ".github/workflows/"+name, "jobs:\n tests:\n steps:\n - run: jest\n") + } + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github/workflows/directory.yml"), 0700)) + files, err = provider.configurationFiles(root) + require.NoError(t, err) + require.Equal(t, []string{".github/workflows/a.yaml", ".github/workflows/z.yml"}, files) + workflows, err := provider.readWorkflows(root) + require.NoError(t, err) + require.Len(t, workflows, len(files)) + for i, workflow := range workflows { + require.Equal(t, files[i], workflow.Path) + } + found, _, err := findTestWorkflows(provider, root, "javascript", "jest") + require.NoError(t, err) + require.Equal(t, files, found) + + writeFile(t, root, files[0], "jobs: [invalid") + _, validationErr := provider.readWorkflows(root) + _, _, instructionErr := findTestWorkflows(provider, root, "javascript", "jest") + require.Error(t, validationErr) + require.EqualError(t, instructionErr, validationErr.Error()) + require.Contains(t, validationErr.Error(), "parse "+filepath.Join(root, files[0])) +} diff --git a/internal/onboard/ci_provider_gitlab.go b/internal/onboard/ci_provider_gitlab.go index 3f429de54..4eead7990 100644 --- a/internal/onboard/ci_provider_gitlab.go +++ b/internal/onboard/ci_provider_gitlab.go @@ -75,15 +75,6 @@ func (g gitLabCI) readWorkflows(root string) ([]ciWorkflow, error) { return config.workflows(), nil } -func (g gitLabCI) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - workflows, err := g.readWorkflows(root) - if err != nil { - return nil, nil, err - } - found, configured := testWorkflowsWithSetup(root, workflows, language, framework) - return found, configured, nil -} - func (gitLabCI) setupStep(step runtimeStep) (ciSetup, bool) { if step.Setup == nil { return ciSetup{}, false diff --git a/internal/onboard/ci_provider_gitlab_test.go b/internal/onboard/ci_provider_gitlab_test.go index 9181736c6..eb154a4a1 100644 --- a/internal/onboard/ci_provider_gitlab_test.go +++ b/internal/onboard/ci_provider_gitlab_test.go @@ -150,7 +150,7 @@ func TestGitLabUnreadableConfigurationStaysUnresolved(t *testing.T) { scope, err := DiscoverValidationScope(root, "jest") require.NoError(t, err) require.NotEmpty(t, scope.Unresolved, "unresolved configuration is never certified") - _, configured, err := gitLabCI{}.findTestWorkflows(root, "javascript", "jest") + _, configured, err := findTestWorkflows(gitLabCI{}, root, "javascript", "jest") require.NoError(t, err) require.Empty(t, configured) }) @@ -525,7 +525,7 @@ func TestGitLabTestWorkflowsNeedTheSetupBeforeTests(t *testing.T) { root := newJestRepository(t, jestWorkflow) writeFile(t, root, ".gitlab-ci.yml", "include: ci/e2e.yml\nunit:\n script:\n"+gitLabSetupEntry(gitLabJestSetup)+" - npx jest\n") writeFile(t, root, "ci/e2e.yml", "late:\n script:\n - npx jest\n"+gitLabSetupEntry(gitLabJestSetup)+"python:\n script:\n"+gitLabSetupEntry(`LANGUAGES="python" API_KEY="$DD_API_KEY"`)+" - npx jest\n") - found, configured, err := gitLabCI{}.findTestWorkflows(root, "javascript", "jest") + found, configured, err := findTestWorkflows(gitLabCI{}, root, "javascript", "jest") require.NoError(t, err) require.Equal(t, []string{".gitlab-ci.yml", "ci/e2e.yml"}, found) require.Equal(t, []string{".gitlab-ci.yml"}, configured) diff --git a/internal/onboard/ci_provider_jenkins.go b/internal/onboard/ci_provider_jenkins.go index 8b331ed52..4ebce2487 100644 --- a/internal/onboard/ci_provider_jenkins.go +++ b/internal/onboard/ci_provider_jenkins.go @@ -109,15 +109,6 @@ func (j jenkins) readWorkflows(root string) ([]ciWorkflow, error) { return workflows, nil } -func (j jenkins) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - workflows, err := j.readWorkflows(root) - if err != nil { - return nil, nil, err - } - found, configured := testWorkflowsWithSetup(root, workflows, language, framework) - return found, configured, nil -} - func (jenkins) setupStep(step runtimeStep) (ciSetup, bool) { if step.Setup == nil { return ciSetup{}, false diff --git a/internal/onboard/ci_provider_test.go b/internal/onboard/ci_provider_test.go index 5703f9fe9..f1044950f 100644 --- a/internal/onboard/ci_provider_test.go +++ b/internal/onboard/ci_provider_test.go @@ -404,7 +404,7 @@ func TestUnknownActiveProviderFails(t *testing.T) { // Provider files parse CI configuration into neutral facts; language decisions // stay behind discoveryLanguage. func TestCIProviderFilesNameNoLanguage(t *testing.T) { - languageguard.CheckNeutral(t, ".", "ci_provider*.go", "ci_providers.go") + languageguard.CheckNeutral(t, ".", "ci_provider*.go", "ci_providers.go", "ci_selection.go", "ci_setup.go", "ci_workflows.go") } // A run without the selected provider's configuration consults no CI, so the diff --git a/internal/onboard/ci_selection.go b/internal/onboard/ci_selection.go new file mode 100644 index 000000000..5da51c548 --- /dev/null +++ b/internal/onboard/ci_selection.go @@ -0,0 +1,240 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "slices" + "strings" + "sync/atomic" + + "github.com/kballard/go-shellquote" +) + +// CIProviderChoice is a user decision: several CI providers contain tests and +// ddtest configures one at a time. No command is suggested until it is made. +type CIProviderChoice struct{ Providers []string } + +func (e *CIProviderChoice) Error() string { + return "Several CI providers contain test commands: " + strings.Join(e.Providers, ", ") + ". Ask the user which provider to install Test Optimization on, then rerun with --ci-provider ." +} + +// activeProvider is the provider selected for this command; empty selects one +// automatically. +var activeProvider atomic.Value + +// UseCIProvider selects the CI provider onboarding and testdrive read for the +// rest of the command, normally the result of SelectCIProvider. Empty selects +// it automatically from the repository again. +func UseCIProvider(name string) { activeProvider.Store(name) } + +// ActiveCIProvider names the provider selected with UseCIProvider, or empty. +func ActiveCIProvider() string { + name, _ := activeProvider.Load().(string) + return name +} + +// CIProviderNames lists the supported --ci-provider values. +func CIProviderNames() []string { + names := make([]string, 0, len(ciProviders)) + for _, provider := range ciProviders { + names = append(names, provider.name()) + } + return names +} + +// UnknownCIProviderError rejects a --ci-provider value no provider registers. +func UnknownCIProviderError(name string) error { + return fmt.Errorf("unsupported --ci-provider %q; supported CI providers: %s", name, strings.Join(CIProviderNames(), ", ")) +} + +func ciProviderNamed(name string) ciProvider { + for _, provider := range ciProviders { + if provider.name() == name { + return provider + } + } + return nil +} + +// defaultCIProvider applies when a repository has no CI configuration: the +// first registered provider, as onboarding behaved before providers were +// selectable. +func defaultCIProvider() ciProvider { return ciProviders[0] } + +// SelectCIProvider resolves the CI provider to configure. A requested provider +// is used as given. Otherwise the only provider with configuration files is +// used without parsing them; with several, the ones whose configuration runs +// tests or leaves steps unresolved are candidates, and more than one candidate +// returns a *CIProviderChoice. A configuration that cannot be parsed never +// fails selection: that provider is not a candidate, and its files stay +// visible as review notes of the selected provider's scope. Empty means no +// provider has configuration files. +func SelectCIProvider(root, requested string) (string, error) { + if requested != "" { + if ciProviderNamed(requested) == nil { + return "", UnknownCIProviderError(requested) + } + return requested, nil + } + var configured []ciProvider + for _, provider := range ciProviders { + // A listing error is reported when that provider is read. + if files, err := provider.configurationFiles(root); err != nil || len(files) > 0 { + configured = append(configured, provider) + } + } + switch len(configured) { + case 0: + return "", nil + case 1: + return configured[0].name(), nil + } + var candidates []string + for _, provider := range configured { + workflows, err := provider.readWorkflows(root) + if err == nil && slices.ContainsFunc(workflows, func(workflow ciWorkflow) bool { return workflowHasTests(root, workflow) }) { + candidates = append(candidates, provider.name()) + } + } + switch len(candidates) { + case 0: + return configured[0].name(), nil + case 1: + return candidates[0], nil + } + return "", &CIProviderChoice{Providers: candidates} +} + +// currentCIProvider is the provider selected with UseCIProvider, otherwise +// the one SelectCIProvider resolves, otherwise the default. +func currentCIProvider(root string) (ciProvider, error) { + name := ActiveCIProvider() + if name == "" { + var err error + if name, err = SelectCIProvider(root, ""); err != nil { + return nil, err + } + } + if name == "" { + return defaultCIProvider(), nil + } + if provider := ciProviderNamed(name); provider != nil { + return provider, nil + } + return nil, UnknownCIProviderError(name) +} + +// displayCIText writes the shared model's checkout expression, which every +// provider's values use, in the selected provider's own syntax, for output. +func displayCIText(text string) string { + reference := activeOrDefaultCIProvider().checkoutReference() + return strings.NewReplacer(ciProjectDirectory, reference, "${{github.workspace}}", reference).Replace(text) +} + +// displayCIError is err with its message in the selected provider's syntax +// (see displayCIText); it still wraps err. +func displayCIError(err error) error { + if err == nil || !strings.Contains(err.Error(), "github.workspace") || activeOrDefaultCIProvider().checkoutReference() == ciProjectDirectory { + return err + } + return displayedCIError{err} +} + +type displayedCIError struct{ err error } + +func (e displayedCIError) Error() string { return displayCIText(e.err.Error()) } +func (e displayedCIError) Unwrap() error { return e.err } + +// activeOrDefaultCIProvider is the selected provider without consulting the +// repository: the default until UseCIProvider selects another. +func activeOrDefaultCIProvider() ciProvider { + if provider := ciProviderNamed(ActiveCIProvider()); provider != nil { + return provider + } + return defaultCIProvider() +} + +// ciProviderFollowUpFlag returns the --ci-provider argument human follow-up +// commands need: only when several providers have configuration in root, so +// a rerun would ask again. +func ciProviderFollowUpFlag(root string) string { + return ciProviderFlag(root) + ciStatementsFollowUp() +} + +// ciStatementsFollowUp repeats the user's CI statements in a human follow-up +// command, which reports are bound to. +func ciStatementsFollowUp() string { + if arguments := CIStatementArguments(); len(arguments) > 0 { + return " " + shellquote.Join(arguments...) + } + return "" +} + +func ciProviderFlag(root string) string { + selected := ActiveCIProvider() + if selected == "" { + return "" + } + configured := 0 + for _, provider := range ciProviders { + if files, err := provider.configurationFiles(root); err != nil || len(files) > 0 { + configured++ + } + } + if configured < 2 { + return "" + } + return " --ci-provider " + selected +} + +// CIProviderDisplayName names the selected CI provider in output. +func CIProviderDisplayName() string { return activeOrDefaultCIProvider().displayName() } + +// workflowHasTests reports whether any active step of workflow may run tests +// of a registered language, or cannot be resolved well enough to tell. +func workflowHasTests(root string, workflow ciWorkflow) bool { + for _, job := range workflow.Jobs { + if disabledCondition(job.If) { + continue + } + if _, _, err := jobMatrix(job); err != nil { + return true // An unresolved matrix can run tests. + } + for _, step := range job.Steps { + if disabledCondition(step.If) { + continue + } + if step.Run == "" && step.ResolutionError != "" { + return true + } + for _, language := range discoveryLanguages { + for _, framework := range language.frameworks() { + resolution := resolveTestStep(root, workflow, job, step, language.name(), framework) + if resolution.Matched || (!resolution.Review && resolution.Reason != "" && stepMayStartTests(root, workflow, job, step, framework)) || namesFramework(step.Run, framework, true) { + return true + } + } + } + } + } + return false +} + +// disabledCondition reports whether condition can never pass, such as false or +// ${{ false }}; anything unresolved may still run. +func disabledCondition(condition string) bool { + if strings.TrimSpace(condition) == "" { + return false + } + active, err := possibleJobCondition(condition, nil) + return err == nil && !active +} + +// ProviderChoiceResult asks the user to choose the CI provider to configure. +func ProviderChoiceResult(choice *CIProviderChoice) AgentResult { + return AgentResult{Status: "needs_input", Phase: "discovery", Code: "ci_provider_required", Reason: choice.Error(), RequiredInput: choice.Providers, NextAction: "ask_user", Instruction: "Ask the user which CI provider should receive Test Optimization. Do not choose one or install on every provider. After their answer, rerun the original command with --ci-provider ."} +} diff --git a/internal/onboard/ci_setup.go b/internal/onboard/ci_setup.go new file mode 100644 index 000000000..bdb2e8359 --- /dev/null +++ b/internal/onboard/ci_setup.go @@ -0,0 +1,503 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "maps" + "slices" + "strings" + + "mvdan.cc/sh/v3/syntax" +) + +// ciSetup is the Datadog Test Optimization setup one CI step runs, described +// in the setup action's terms. Each provider recognizes its own mechanism +// (see ciProvider.setupStep); shared code never inspects the mechanism. +type ciSetup struct { + // Reference names what the step runs, such as the action and its ref. + Reference string + // Pinned reports a reference that selects a release. + Pinned bool + // Supported reports a release ddtest validates, with its integrity check + // where the provider has one. + Supported bool + // DefaultLatest reports that an unset tracer version input installs the + // latest release; otherwise the mechanism's metadata defines the default. + DefaultLatest bool + // Problem is why the setup cannot work where it runs, as the provider + // found while reading it; setupProblem reports it first. + Problem string + // shaped reports a setup step in a form the provider recognizes entirely. + shaped bool + // fallback reports ddtest's setup without Bash or curl (see + // fallbackSetupAt), which every provider validates the same way. + fallback bool + // after are shell statements that run only with the setup, inside its + // guard. + after []*syntax.Stmt + // before are shell statements a setup snippet runs before itself, + // whether or not it installs anything (see snippetSetupAt). + before []string + // shellLocal reports a setup whose exports reach only the later + // commands of the shell it runs in, such as a setup snippet in a + // Jenkinsfile sh step: a step of its job in another shell does not have + // them (see ciShellOrigin). + shellLocal bool + // Inputs are the installer inputs keyed by the setup action's input + // names: languages, api_key, site and -tracer-version. Values + // are verbatim; unresolved references stay as written. + Inputs map[string]string + // Floating reports a mechanism that cannot pin the tracer: it installs + // the latest release when the build runs, unless the project's own + // dependencies already pin one. Each language decides which applies and + // reports the floating release as a caveat. + Floating bool + // WorkspaceRoot reports a mechanism that configures the tracer from the + // checkout root with the build agent's own toolchain, so a tracer that + // joins the project's dependencies must join the root ones. + WorkspaceRoot bool + // Confirmations are the facts (--ci-confirm keys, see ciProviderFacts) + // the user must state before ddtest certifies this setup. + Confirmations []string + // Caveats are what ddtest could not check about the mechanism, for + // final_response. + Caveats []string + // Beside names another Datadog setup mechanism the build enables for + // the same language in other jobs that share its workspace, such as the + // Datadog Jenkins plugin beside a container stage's setup snippet; + // languages whose tracer joins the project's dependencies check that it + // cannot change the release this setup installs. + Beside string + // Exports are the loader variables the mechanism sets for the steps + // after it besides the action's DD_TRACE_PACKAGE and DD_TRACE_ESM_IMPORT, + // such as the Datadog Jenkins plugin's DD_TRACE_PATH. A test step's + // NODE_OPTIONS can trust only the variables its own setup exports. + Exports []string +} + +// ciSetupRequest is the setup one language asks a provider for: the +// provider supplies the step, the language its own values. +type ciSetupRequest struct { + // Language is the installer's token for the language (languages input). + Language string + // Site is the Datadog site the setup reports to. + Site string + // TracerInput names the exact tracer release input, such as + // js-tracer-version, and TracerVersion its value. + TracerInput, TracerVersion string + // After are shell commands that must follow the setup and precede the + // tests in the same job, such as an ESM loader registration. + After []string + // Notes are language rules the instructions add after the step. + Notes []string + // Service is the service name a setup that names one reports under: the + // project's own name, empty when the language reads none. + Service string + // Framework is the framework whose tests the setup is for. + Framework string + // Jobs are the jobs the setup is for, when the provider sets its jobs up + // with several mechanisms (see runtimeJob.SetupProvider); empty for all. + Jobs []string + // Shared are the job definitions of these jobs whose other invocations + // run no tests (see circleSharedJobs): the setup is enabled only for the + // invocations that run the tests. + Shared []ciSharedJob +} + +// ciSetupTerms are how a provider without the shared workflow syntax names +// its setup in the languages' setup notes, which refer to the setup's place +// in a job and to how a later command keeps a variable it exports. +type ciSetupTerms struct { + // Installer names the setup mechanism after "the", such as "script". + Installer string + // Entry names the setup's place in a job, such as "the entry", and + // SetupEntry the same place where the setup must be named, such as "the + // setup entry". + Entry, SetupEntry string + // Later names what the setup's exports reach, such as "later commands of + // the job". + Later string + // Variables names where a job declares its variables, such as "the job's + // variables". + Variables string + // Persist describes, in a sentence, running a shell command after the + // setup so that later test commands see what it exports; PersistEntry + // writes it as an entry of the job's configuration (a YAML list item). + Persist, PersistEntry func(command string) string + // MatrixExclusion says how matrix entries leave a job, such as "with + // rules (when: never on their matrix variables)". + MatrixExclusion string + // SharedMatrix names a matrix whose entries share one job's commands, + // such as "a parallel:matrix shares the script". + SharedMatrix string + // ImageGuard writes an entry that runs command only where the job's + // image, written as template, is image; empty when the template selects + // no image through a variable or parameter. + ImageGuard func(template, image, command string) string + // Beside names the provider's other setup mechanism that the same build + // runs for the language in other jobs of its workspace (see + // runtimeJob.SetupProvider), such as the Datadog Jenkins plugin beside + // container stages' setup snippet; empty when there is none. + Beside string + // SecretStore names where the provider keeps the DD_API_KEY a job reads + // without writing it, such as "a masked GitLab CI/CD variable", for + // setups written by hand. + SecretStore string + // KeyBinding reports a provider that gives a job the stored key only + // where its configuration binds it (a Jenkins credential), not merely + // because the secret store holds it: a hand-written setup without + // DD_API_KEY in its environment then reports nothing. + KeyBinding bool + // Fallback reports the setup without Bash or curl (see imageFallback), + // which installs the tracer with the job's own tools: languages describe + // what it exports. + Fallback bool + // Plugin reports a setup that is a CI server plugin, enabled in the + // configuration, that installs the tracers on the build agent and sets + // their variables for every step (the Datadog Jenkins plugin), instead of + // an installer the job runs: languages describe that mechanism. + Plugin bool +} + +// ciComponent is a reusable CI component a language knows, such as a +// runtime or dependency installer published for one provider, modeled in the +// provider's own syntax so the provider reads it like the repository's +// configuration. +type ciComponent struct { + // Name is the component's registry name, such as an orb's + // namespace/name. + Name string + // Majors are the major versions the model describes. + Majors []string + // Definition is the model: for CircleCI an inline orb's commands, jobs + // and executors. + Definition string +} + +// ciSetupTarget is one test step that needs the setup before it. +type ciSetupTarget struct { + Workflow, Job string + // Step indexes the job's steps. + Step int + // After overrides the request's commands for this step. + After []string +} + +// datadogSetup recognizes the Datadog setup a step runs, whichever provider +// read it. +func datadogSetup(step runtimeStep) (ciSetup, bool) { + if step.Setup != nil { + return *step.Setup, true + } + for _, provider := range ciProviders { + if setup, ok := provider.setupStep(step); ok { + return setup, true + } + } + return ciSetup{}, false +} + +// setupProviderFor is the provider whose setup mechanism job uses: its +// own, unless the job runs another of its mechanisms (see +// runtimeJob.SetupProvider), whose names, inputs and instructions then +// apply. +func setupProviderFor(provider ciProvider, job runtimeJob) ciProvider { + if job.SetupProvider != nil { + return job.SetupProvider + } + return provider +} + +// jobSetups are the jobs that set Test Optimization up with one of a +// provider's mechanisms. +type jobSetups struct { + provider ciProvider + jobs []string +} + +// testJobSetups groups the jobs that run framework's tests by the setup +// mechanism each uses (see setupProviderFor), the provider's own first; a +// provider with one mechanism, or no such job, gives that one alone. +func testJobSetups(root string, provider ciProvider, language, framework string) []jobSetups { + groups := []jobSetups{{provider: provider}} + workflows, err := readProviderWorkflows(provider, root) + if err != nil { + return groups + } + for _, workflow := range workflows { + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + if disabledCondition(job.If) || !slices.ContainsFunc(job.Steps, func(step runtimeStep) bool { + return resolveTestStep(root, workflow, job, step, language, framework).Matched + }) { + continue + } + mechanism := setupProviderFor(provider, job) + i := slices.IndexFunc(groups, func(group jobSetups) bool { return group.provider == mechanism }) + if i < 0 { + groups, i = append(groups, jobSetups{provider: mechanism}), len(groups) + } + groups[i].jobs = append(groups[i].jobs, workflow.Path+" / "+name) + } + } + if len(groups) > 1 && len(groups[0].jobs) == 0 { + groups = groups[1:] + } + return groups +} + +// setupProviderNames names the setup mechanisms of groups. +func setupProviderNames(groups []jobSetups) string { + names := make([]string, 0, len(groups)) + for _, group := range groups { + if name := group.provider.setupName(); !slices.Contains(names, name) { + names = append(names, name) + } + } + return strings.Join(names, " or the ") +} + +// setupSelectsLanguage reports whether setup installs the tracer of the +// language named by its languages input, after resolving row's matrix values. +func setupSelectsLanguage(setup ciSetup, row map[string]any, language string) (bool, error) { + languages, err := runtimeValue(setup.Inputs["languages"], row) + if err != nil { + return false, err + } + return languages == "all" || slices.Contains(strings.Fields(languages), language), nil +} + +// CIProviderSetupName names the selected CI provider's setup mechanism when +// it is not written in the shared workflow syntax; empty otherwise, where +// languages name the setup action themselves. +func CIProviderSetupName() string { + if provider := activeOrDefaultCIProvider(); !provider.sharedSyntax() { + return provider.setupName() + } + return "" +} + +// CIProviderTracerSetting is how the selected CI provider's setup pins +// the language's tracer release, such as js-tracer-version: 5.133.0, or +// DD_SET_TRACER_VERSION_JS="5.133.0" in the Buildkite setup snippet. +func CIProviderTracerSetting(language, version string) string { + provider := activeOrDefaultCIProvider() + input := provider.setupInput(language + "-tracer-version") + switch provider.name() { + case buildkiteProvider: + return input + `="` + version + `"` + case jenkinsProvider: + return "release " + version + " where the CI setup can pin it (the " + input + ")" + } + return input + ": " + version +} + +// CIProviderSiteSetting names site in place of the example Datadog site the +// manual setup instructions show, which they render for the default site: +// empty for that default, which needs no substitution. +func CIProviderSiteSetting(site string) string { + if site == "" || site == defaultDatadogSite { + return "" + } + return "the Datadog site " + site + " in place of the example site " + defaultDatadogSite +} + +// CIProviderAPIKeyAction is the person's step that stores the Datadog API key +// for the selected CI provider. +func CIProviderAPIKeyAction(skipped bool) string { + return activeOrDefaultCIProvider().apiKeyAction(skipped) +} + +// ciAPIKeyActions is a provider whose jobs set Test Optimization up with +// several mechanisms (see runtimeJob.SetupProvider), each receiving the +// API key its own way. +type ciAPIKeyActions interface { + // apiKeyActions are the person's steps for the mechanisms named by + // references (ciSetup.Reference of the validated jobs' setups). + apiKeyActions(references []string, skipped bool) string +} + +// CIProviderAPIKeyActionFor is CIProviderAPIKeyAction for the setups of +// check's jobs, on a provider whose jobs use several mechanisms. +func CIProviderAPIKeyActionFor(check *RuntimeCheck, skipped bool) string { + provider := activeOrDefaultCIProvider() + several, ok := provider.(ciAPIKeyActions) + if !ok && check != nil { + // Jobs on the setup without Bash or curl skip it without the key. + fallback := slices.ContainsFunc(check.Jobs, func(job RuntimeFinding) bool { return job.Action == fallbackSetupName }) + others := slices.ContainsFunc(check.Jobs, func(job RuntimeFinding) bool { return job.Action != "" && job.Action != fallbackSetupName }) + switch { + case fallback && others: + return provider.apiKeyAction(skipped) + " " + newImageFallback(provider, "", "").apiKeyAction(skipped) + case fallback: + return newImageFallback(provider, "", "").apiKeyAction(skipped) + } + } + if !ok || check == nil { + return provider.apiKeyAction(skipped) + } + var references []string + for _, job := range check.Jobs { + if job.Action != "" { + references = append(references, job.Action) + } + } + return several.apiKeyActions(references, skipped) +} + +// CIProviderUnsupportedError reports a language without CI setup support on +// the selected provider, or (Reason) a repository whose every test job is +// outside what the provider's setup mechanism instruments. ddtest +// prescribes nothing for that combination. +type CIProviderUnsupportedError struct { + Language, Framework, Provider string + // Reason names the jobs the provider's setup cannot instrument, and + // why; Setup names that setup. + Reason, Setup string + // HandWritten names the jobs that set up Test Optimization by hand in + // a way ddtest recognizes but does not validate for Language yet (see + // handWrittenSetup.Unvalidated), each with its setup. + HandWritten []string +} + +func (e *CIProviderUnsupportedError) Error() string { + if len(e.HandWritten) > 0 { + return fmt.Sprintf("ddtest cannot validate %s CI for %s on %s in this repository: ddtest does not validate hand-written %s setups yet, and these CI jobs set up Test Optimization by hand: %s ddtest neither certifies those setups nor replaces them with the %s. Keep them as they are, and do not add the %s to those jobs, remove their setup or change how they report; what a setup lacks is for the user to decide. ddtest will not prescribe CI edits or validate this CI configuration.", e.Framework, e.Language, e.Provider, handWrittenLanguageName(e.Language), strings.TrimSuffix(strings.Join(e.HandWritten, "; "), ".")+".", e.Setup, e.Setup) + } + if e.Reason != "" { + return fmt.Sprintf("ddtest cannot set up %s CI for %s on %s in this repository: every CI job that runs the tests is outside what the %s instruments. %s The user's options are to run those tests where the %s instruments them, or %s. ddtest will not prescribe CI edits or validate this CI configuration as it is.", e.Framework, e.Language, e.Provider, e.Setup, e.Reason, e.Setup, handWrittenSetupOption(e.Language, e.Setup)) + } + return fmt.Sprintf("ddtest does not support %s CI setup for %s on %s; it will not prescribe CI edits or validate this CI configuration. Rerun with --ci-provider to choose another provider.", e.Framework, e.Language, e.Provider) +} + +// languageSupportsCIProvider reports whether language sets up Test +// Optimization on provider. Unregistered languages support none. +func languageSupportsCIProvider(language string, provider ciProvider) bool { + registered := discoveryLanguageNamed(language) + return registered != nil && registered.supportsCIProvider(provider.name()) +} + +// CheckCIProviderSupport returns a *CIProviderUnsupportedError when the +// selected CI provider has no setup support for language's framework. +func CheckCIProviderSupport(root, language, framework string) error { + provider, err := currentCIProvider(root) + if err != nil { + return err + } + // Without the provider's configuration there is no CI to consult or set up. + if files, err := provider.configurationFiles(root); err == nil && len(files) == 0 { + return nil + } + return providerSupportError(root, provider, language, framework) +} + +// providerSupportError is CheckCIProviderSupport's result once the +// provider's configuration exists. A hand-written setup ddtest recognizes +// but does not validate for language comes first, whatever the provider +// supports, so it is never replaced or certified. +func providerSupportError(root string, provider ciProvider, language, framework string) error { + if jobs := unvalidatedHandWrittenJobs(root, provider, language, framework); len(jobs) > 0 { + return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName(), Setup: provider.setupName(), HandWritten: jobs} + } + if !languageSupportsCIProvider(language, provider) { + return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName()} + } + if reason := unsupportedTestJobs(root, provider, language, framework); reason != "" { + return &CIProviderUnsupportedError{Language: language, Framework: framework, Provider: provider.displayName(), Reason: reason, Setup: provider.setupName()} + } + return nil +} + +// unsupportedTestJobs explains, when every CI job that runs framework's +// tests is one the provider's setup mechanism cannot instrument (see +// runtimeJob.SetupUnsupported), why; empty when any can, or none runs them. +func unsupportedTestJobs(root string, provider ciProvider, language, framework string) string { + workflows, err := readProviderWorkflows(provider, root) + if err != nil { + return "" + } + var reasons []string + for _, workflow := range workflows { + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + if disabledCondition(job.If) { + continue + } + tests := slices.ContainsFunc(job.Steps, func(step runtimeStep) bool { + resolution := resolveTestStep(root, workflow, job, step, language, framework) + return !resolution.Review && (resolution.Matched || resolution.Reason != "" && step.ResolutionError == "" && stepMayStartTests(root, workflow, job, step, framework)) + }) + if !tests { + continue + } + // A job that sets up the tracer by hand needs no provider setup. + if job.SetupUnsupported == "" || firstHandWrittenSetup(root, workflow, job, language, framework, "") != nil { + return "" + } + reasons = append(reasons, workflow.Path+" / "+name+": "+job.SetupUnsupported+".") + } + } + return strings.Join(reasons, " ") +} + +// ProviderUnsupportedResult reports a language/provider combination ddtest +// does not set up. +func ProviderUnsupportedResult(err *CIProviderUnsupportedError) AgentResult { + result := AgentResult{Status: "unsupported", Phase: "onboarding", Code: "ci_provider_unsupported", Reason: err.Error(), NextAction: "report_result"} + switch { + case len(err.HandWritten) > 0: + result.Code = "ci_setup_unsupported" + result.Instruction = "Report this result to the user with its reason and stop. Keep the hand-written setups as they are: do not add the " + err.Setup + ", edit CI, add or change a tracer, or remove a setup; those are the user's decisions, and ddtest cannot validate hand-written " + handWrittenLanguageName(err.Language) + " setups yet." + case err.Reason != "": + result.Code = "ci_setup_unsupported" + result.Instruction = "Report this result to the user with its reason and stop. Do not move tests between agents, edit CI, add a tracer or set up Test Optimization by hand: those are the user's decisions." + if validatesHandWrittenSetups(err.Language) { + result.Instruction += " Once the user sets it up by hand, rerun onboarding, which checks that setup." + } else { + result.Instruction += " ddtest does not validate hand-written " + handWrittenLanguageName(err.Language) + " setups yet." + } + } + return result +} + +// CIProviderEditsSetup reports whether onboard --apply writes the selected +// CI provider's setup; otherwise every language gets instructions. +func CIProviderEditsSetup() bool { return activeOrDefaultCIProvider().editsSetup() } + +// providerAutomaticSetup reports whether onboard --apply writes framework's +// setup on provider in root, reporting to site (empty: any): the provider +// edits its configuration and the language writes that framework's setup +// there. A plugin setup applies only where the provider can make the edit +// and leave it complete for site; otherwise the instructions do. +func providerAutomaticSetup(root string, provider ciProvider, language discoveryLanguage, framework, site string) bool { + return provider.editsSetup() && language.automaticProviderSetup(framework) && (!provider.setupTerms().Plugin || pluginSetupApplies(root, framework, site)) +} + +// ciConfirmationProblem lists the provider facts the user has not +// confirmed (--ci-confirm) that this entry's setup depends on, and how to +// confirm them; empty when every one is confirmed. +func ciConfirmationProblem(provider ciProvider, setup ciSetup) string { + var missing []ciFactDefinition + for _, definition := range ciProviderFacts[provider.name()] { + if _, stated := statedFact(definition.Key); slices.Contains(setup.Confirmations, definition.Key) && !stated { + missing = append(missing, definition) + } + } + if len(missing) == 0 { + return "" + } + var questions, flags []string + for _, fact := range missing { + flag := fact.Key + if fact.Values != "" { + flag += "=" + fact.Values + } + questions = append(questions, strings.TrimSpace(fact.Key+": "+fact.Question+" "+fact.Remedy)) + flags = append(flags, "--ci-confirm "+flag) + } + return "The " + provider.setupName() + " setup depends on facts no CI configuration shows, so ddtest cannot certify it until the user confirms them. Ask the user; never assume or check them from this machine. " + strings.Join(questions, " ") + " Then rerun testdrive adding " + strings.Join(flags, " ") + " with the values they confirmed." +} diff --git a/internal/onboard/ci_statements.go b/internal/onboard/ci_statements.go index cee260c29..639b9eebd 100644 --- a/internal/onboard/ci_statements.go +++ b/internal/onboard/ci_statements.go @@ -369,27 +369,6 @@ func fillStatedRuntimes(workflows []ciWorkflow) { } } -// readProviderWorkflows reads a provider's configuration with the user's -// runtime statements recorded on its jobs. -func readProviderWorkflows(provider ciProvider, root string) ([]ciWorkflow, error) { - workflows, err := provider.readWorkflows(root) - if err == nil { - fillStatedRuntimes(workflows) - checkFallbackPrerequisites(provider, workflows) - checkout := []string{ciProjectDirectory, "${{github.workspace}}"} - if name := provider.checkoutVariable(); name != "" { - checkout = append(checkout, "$"+name, "${"+name+"}") - } - for _, workflow := range workflows { - for name, job := range workflow.Jobs { - job.Checkout = append(slices.Clone(checkout), job.Checkout...) - workflow.Jobs[name] = job - } - } - } - return workflows, err -} - // runtimeHost describes where a job whose configuration selects no runtime // runs on a matrix row (nil: the job without a row), for a person who knows // that host's runtime; empty when ddtest does not take the user's statement diff --git a/internal/onboard/ci_workflows.go b/internal/onboard/ci_workflows.go new file mode 100644 index 000000000..7e8d7da32 --- /dev/null +++ b/internal/onboard/ci_workflows.go @@ -0,0 +1,159 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "maps" + "slices" +) + +type runDefaults struct { + WorkingDirectory string `yaml:"working-directory"` + Shell string `yaml:"shell"` +} + +type ciDefaults struct { + Run runDefaults `yaml:"run"` +} + +type ciWorkflow struct { + On any `yaml:"on"` + Path string + Defaults ciDefaults `yaml:"defaults"` + Env map[string]string `yaml:"env"` + Jobs map[string]runtimeJob `yaml:"jobs"` + // Files are the other repository files the provider read for this + // workflow, such as included configuration. + Files []string `yaml:"-"` + // Review lists configuration the provider read as outside every test + // path, reported without blocking. + Review []string `yaml:"-"` + // Caveats summarize that configuration, one line per group. + Caveats []string `yaml:"-"` +} + +// readWorkflows reads the selected provider's CI configuration. +func readWorkflows(root string) ([]ciWorkflow, error) { + provider, err := currentCIProvider(root) + if err != nil { + return nil, err + } + return readProviderWorkflows(provider, root) +} + +// WithCIProviderDefaults adds the variables the selected CI provider sets in +// every job, without overriding the configuration's own. +func WithCIProviderDefaults(env map[string]string) map[string]string { + return mergeEnvironment(activeOrDefaultCIProvider().environment(), env) +} + +// CIProviderConfigurationFiles lists the selected CI provider's configuration +// files in root without parsing them. +func CIProviderConfigurationFiles(root string) ([]string, error) { + return activeOrDefaultCIProvider().configurationFiles(root) +} + +// CIConfigurationFiles lists every repository file the selected CI +// provider's configuration is read from, included files too. A +// configuration that cannot be read lists its configuration files only. +func CIConfigurationFiles(root string) []string { + provider := activeOrDefaultCIProvider() + files, _ := provider.configurationFiles(root) + workflows, err := provider.readWorkflows(root) + if err != nil { + return files + } + for _, workflow := range workflows { + for _, file := range append([]string{workflow.Path}, workflow.Files...) { + if !slices.Contains(files, file) { + files = append(files, file) + } + } + } + return files +} + +// testWorkflowsWithSetup lists the workflows whose jobs run framework's tests +// and those where an active setup selecting language precedes every test step. +func testWorkflowsWithSetup(root string, workflows []ciWorkflow, language, framework string) (found, configured []string) { + token := setupActionLanguage(language) + for _, workflow := range workflows { + tests, complete := false, true + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + if disabledCondition(job.If) { + continue + } + setup := false + for i, step := range job.Steps { + if current, ok := datadogSetup(step); ok { + selected, err := setupSelectsLanguage(current, nil, token) + setup = setup || err == nil && selected + } + resolution := resolveTestStep(root, workflow, job, step, language, framework) + if resolution.Review || !resolution.Matched && (resolution.Reason == "" || !stepMayStartTests(root, workflow, job, step, framework)) { + continue + } + tests = true + // A complete setup the job writes by hand counts as its own. + handWritten := !setup && resolution.Matched && resolution.Reason == "" && func() bool { + current := handWrittenCISetup(root, workflow, job, i, nil, language, framework, "") + return current != nil && current.complete() + }() + complete = complete && (setup || handWritten) && resolution.Reason == "" + } + } + if tests { + found = append(found, workflow.Path) + if complete { + configured = append(configured, workflow.Path) + } + } + } + return found, configured +} + +// readProviderWorkflows reads a provider's configuration with the user's +// runtime statements recorded on its jobs. +func readProviderWorkflows(provider ciProvider, root string) ([]ciWorkflow, error) { + workflows, err := provider.readWorkflows(root) + if err == nil { + fillStatedRuntimes(workflows) + checkFallbackPrerequisites(provider, workflows) + checkout := []string{ciProjectDirectory, "${{github.workspace}}"} + if name := provider.checkoutVariable(); name != "" { + checkout = append(checkout, "$"+name, "${"+name+"}") + } + for _, workflow := range workflows { + for name, job := range workflow.Jobs { + job.Checkout = append(slices.Clone(checkout), job.Checkout...) + workflow.Jobs[name] = job + } + } + } + return workflows, err +} + +// ciTestWorkflowFinder overrides instruction discovery where a provider needs +// literal steps instead of the resolved workflow inspection used by default. +type ciTestWorkflowFinder interface { + findTestWorkflows(root, language, framework string) ([]string, []string, error) +} + +// findTestWorkflows applies the shared test/setup classification after a provider +// parses its configuration. GitHub's manual instructions retain their literal +// inspection; full discovery and validation still resolve composite actions. +func findTestWorkflows(provider ciProvider, root, language, framework string) ([]string, []string, error) { + if finder, ok := provider.(ciTestWorkflowFinder); ok { + return finder.findTestWorkflows(root, language, framework) + } + workflows, err := provider.readWorkflows(root) + if err != nil { + return nil, nil, err + } + found, configured := testWorkflowsWithSetup(root, workflows, language, framework) + return found, configured, nil +} diff --git a/internal/onboard/discovery.go b/internal/onboard/discovery.go index 940f675c2..200343aeb 100644 --- a/internal/onboard/discovery.go +++ b/internal/onboard/discovery.go @@ -13,31 +13,6 @@ type workflowDiscovery struct { Workflows, Configured, Unresolved, Review []string } -type runDefaults struct { - WorkingDirectory string `yaml:"working-directory"` - Shell string `yaml:"shell"` -} - -type ciDefaults struct { - Run runDefaults `yaml:"run"` -} - -type ciWorkflow struct { - On any `yaml:"on"` - Path string - Defaults ciDefaults `yaml:"defaults"` - Env map[string]string `yaml:"env"` - Jobs map[string]runtimeJob `yaml:"jobs"` - // Files are the other repository files the provider read for this - // workflow, such as included configuration. - Files []string `yaml:"-"` - // Review lists configuration the provider read as outside every test - // path, reported without blocking. - Review []string `yaml:"-"` - // Caveats summarize that configuration, one line per group. - Caveats []string `yaml:"-"` -} - func findWorkflows(root, language, name string) (workflowDiscovery, error) { var result workflowDiscovery workflows, err := readWorkflows(root) diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index be3e8f5a7..97da64e42 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -65,7 +65,7 @@ func run(output io.Writer, agent *agentHandoff) error { // onboardCI prints the onboarding guidance for the selected CI provider. func onboardCI(output io.Writer, provider ciProvider, repositoryRoot, language, name string, agent *agentHandoff) error { - workflows, configured, err := provider.findTestWorkflows(repositoryRoot, language, name) + workflows, configured, err := findTestWorkflows(provider, repositoryRoot, language, name) if err != nil { return err } diff --git a/internal/onboard/python.go b/internal/onboard/python.go index d4cef1fef..55e083f51 100644 --- a/internal/onboard/python.go +++ b/internal/onboard/python.go @@ -16,8 +16,8 @@ import ( "strings" "time" - "github.com/DataDog/ddtest/internal/environment" "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/testcommand" "github.com/kballard/go-shellquote" "go.yaml.in/yaml/v3" ) @@ -129,7 +129,7 @@ func pythonStepCandidate(root string, workflow ciWorkflow, job runtimeJob, step if resolution.Review || resolution.Reason == "" { return false } - if environment.LooksLikeTestCommand(root, step.Run, "python", "pytest") || pythonTestReference.MatchString(step.Run) { + if testcommand.Matches(root, step.Run, "python", "pytest") || pythonTestReference.MatchString(step.Run) { return true // Conditional shell code remains unresolved, never replayed. } commands, _ := pythonStepCommands(step.Run) @@ -780,7 +780,7 @@ func resolvePythonStep(root string, workflow ciWorkflow, job runtimeJob, step ru result.ReviewReason = joinDiagnostic(result.ReviewReason, words[0]+" runs no tests; it is outside this validation.") continue } - if !environment.LooksLikeTestCommand(root, shellquote.Join(words...), "python", "pytest") && (slices.Contains(words, "tox") || slices.Contains(words, "nox")) { + if !testcommand.Matches(root, shellquote.Join(words...), "python", "pytest") && (slices.Contains(words, "tox") || slices.Contains(words, "nox")) { return commandResolution{Review: true, Reason: "This wrapper does not identify a pytest test environment; review it separately."} } return unresolvedCommand("Unresolved Python entry point: " + shellquote.Join(words...) + ". For tox/nox or other wrappers, activate the selected test environment and pass its Python/pytest command explicitly.") diff --git a/internal/onboard/ruby.go b/internal/onboard/ruby.go index 1872121e0..da943b4b0 100644 --- a/internal/onboard/ruby.go +++ b/internal/onboard/ruby.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/testcommand" "github.com/kballard/go-shellquote" "mvdan.cc/sh/v3/syntax" ) @@ -38,7 +38,7 @@ func resolveRubyStep(root string, workflow ciWorkflow, job runtimeJob, step runt return resolution } parts, parseErr := staticCommands(step.Run) - matched := environment.LooksLikeTestCommand(filepath.Join(root, directory), step.Run, "ruby", framework) + matched := testcommand.Matches(filepath.Join(root, directory), step.Run, "ruby", framework) if parseErr == nil && len(parts) == 1 { words := parts[0] if len(words) > 2 && filepath.Base(words[0]) == "bundle" && words[1] == "exec" { @@ -56,7 +56,7 @@ func resolveRubyStep(root string, workflow ciWorkflow, job runtimeJob, step runt } } if len(words) > 0 && filepath.Base(words[0]) == "rake" { - if !environment.LooksLikeTestCommand(filepath.Join(root, directory), step.Run, "ruby", framework) { + if !testcommand.Matches(filepath.Join(root, directory), step.Run, "ruby", framework) { return commandResolution{Review: true, Reason: "Ruby maintenance task is outside test validation."} } if resolution, done := resolveRubyRakeTargets(root, directory, step.Run, parts[0], words, framework); done { diff --git a/internal/onboard/scripts.go b/internal/onboard/scripts.go index 6eaf36dee..609c38bc4 100644 --- a/internal/onboard/scripts.go +++ b/internal/onboard/scripts.go @@ -16,7 +16,7 @@ import ( "strings" "unicode" - "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/testcommand" "github.com/kballard/go-shellquote" ) @@ -152,7 +152,7 @@ func resolveTestStep(root string, workflow ciWorkflow, job runtimeJob, step runt return commandResolution{} } if !resolvesTestSteps(language, framework) { - return commandResolution{Matched: environment.LooksLikeTestCommand(root, step.Run, language, framework)} + return commandResolution{Matched: testcommand.Matches(root, step.Run, language, framework)} } // GitHub expands these before the shell runs; they are not shell syntax. if expression := workflowExpression.FindString(step.Run); expression != "" { diff --git a/internal/environment/github_workflows.go b/internal/testcommand/match.go similarity index 63% rename from internal/environment/github_workflows.go rename to internal/testcommand/match.go index e9231dbd4..8e516b7b1 100644 --- a/internal/environment/github_workflows.go +++ b/internal/testcommand/match.go @@ -3,150 +3,29 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2026 Datadog, Inc. -package environment +// Package testcommand recognizes test entry points without executing repository code. +package testcommand import ( "encoding/json" - "fmt" "os" "path/filepath" "slices" - "sort" "strings" "github.com/DataDog/ddtest/internal/maven" "github.com/kballard/go-shellquote" "github.com/pelletier/go-toml/v2" - "go.yaml.in/yaml/v3" ) -const githubAction = "datadog/test-visibility-github-action" - -func gitHubWorkflowPaths(repositoryRoot string) ([]string, error) { - directory := filepath.Join(repositoryRoot, ".github", "workflows") - entries, err := os.ReadDir(directory) - if os.IsNotExist(err) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("find GitHub Actions workflows: %w", err) - } - var paths []string - for _, entry := range entries { - if !entry.IsDir() && (filepath.Ext(entry.Name()) == ".yml" || filepath.Ext(entry.Name()) == ".yaml") { - paths = append(paths, filepath.Join(directory, entry.Name())) - } - } - return paths, nil -} - -// FindTestWorkflows returns the workflows that run the selected framework and -// the subset with an active Datadog action before every matching test step. -func (g *GitHub) FindTestWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { - var workflows []string - var configured []string - paths, err := gitHubWorkflowPaths(repositoryRoot) - if err != nil { - return nil, nil, err - } - for _, path := range paths { - contents, err := os.ReadFile(path) - if err != nil { - return nil, nil, fmt.Errorf("read %s: %w", path, err) - } - foundTests, isConfigured, err := inspectTestJobs(repositoryRoot, contents, language, name) - if err != nil { - return nil, nil, fmt.Errorf("parse %s: %w", path, err) - } - if !foundTests { - continue - } - - relativePath, err := filepath.Rel(repositoryRoot, path) - if err != nil { - return nil, nil, fmt.Errorf("make workflow path relative: %w", err) - } - relativePath = filepath.ToSlash(relativePath) - workflows = append(workflows, relativePath) - if isConfigured { - configured = append(configured, relativePath) - } - } - - sort.Strings(workflows) - sort.Strings(configured) - return workflows, configured, nil -} - -func inspectTestJobs(root string, contents []byte, language, name string) (foundTests, configured bool, err error) { - type defaults struct { - Run struct { - WorkingDirectory string `yaml:"working-directory"` - } `yaml:"run"` - } - var workflow struct { - Defaults defaults `yaml:"defaults"` - Jobs map[string]struct { - Defaults defaults `yaml:"defaults"` - Steps []struct { - Run string `yaml:"run"` - Uses string `yaml:"uses"` - If string `yaml:"if"` - WorkingDirectory string `yaml:"working-directory"` - With struct { - Languages string `yaml:"languages"` - } `yaml:"with"` - } `yaml:"steps"` - } `yaml:"jobs"` - } - if err := yaml.Unmarshal(contents, &workflow); err != nil { - return false, false, err - } - actionLanguage := language - if language == "javascript" { - actionLanguage = "js" - } - for _, job := range workflow.Jobs { - jobConfigured := false - for _, step := range job.Steps { - condition := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(step.If), "${{"), "}}")) - if condition == "false" { - continue - } - directory := step.WorkingDirectory - if directory == "" { - directory = job.Defaults.Run.WorkingDirectory - } - if directory == "" { - directory = workflow.Defaults.Run.WorkingDirectory - } - if !filepath.IsAbs(directory) { - directory = filepath.Join(root, directory) - } - if looksLikeTestJob(directory, step.Run, language, name, 0) { - foundTests = true - if !jobConfigured { - return true, false, nil - } - } - action, _, hasVersion := strings.Cut(strings.ToLower(step.Uses), "@") - if action == githubAction && hasVersion && (condition == "" || condition == "true" || condition == "success()") && !strings.Contains(step.With.Languages, "${{") { - languages := strings.Fields(step.With.Languages) - jobConfigured = jobConfigured || step.With.Languages == "all" || slices.Contains(languages, actionLanguage) - } - } - } - return foundTests, foundTests, nil -} - -// LooksLikeTestCommand recognizes literal framework invocations without running project code. -func LooksLikeTestCommand(root, commands, language, name string) bool { - return looksLikeTestJob(root, commands, language, name, 0) +// Matches recognizes literal framework invocations without running project code. +func Matches(root, commands, language, name string) bool { + return matches(root, commands, language, name, 0) } // Resolve literal invocations without running project code. Limit script recursion // so cycles in package.json cannot hang onboarding. -func looksLikeTestJob(root, commands, language, name string, depth int) bool { +func matches(root, commands, language, name string, depth int) bool { if depth > 10 { return false } @@ -158,7 +37,7 @@ func looksLikeTestJob(root, commands, language, name string, depth int) bool { // Handle simple shell command lists. Leave quoted shell expressions opaque. if !strings.ContainsAny(command, "\"'`") && strings.ContainsAny(command, ";&|") { for _, part := range strings.FieldsFunc(command, func(r rune) bool { return r == ';' || r == '&' || r == '|' }) { - if looksLikeTestJob(root, part, language, name, depth+1) { + if matches(root, part, language, name, depth+1) { return true } } @@ -239,7 +118,7 @@ func looksLikeTestJob(root, commands, language, name string, depth int) bool { continue } if script, ok := manifest.Scripts[args[0]]; ok { - if looksLikeTestJob(root, script, language, name, depth+1) { + if matches(root, script, language, name, depth+1) { return true } continue diff --git a/internal/testcommand/match_test.go b/internal/testcommand/match_test.go new file mode 100644 index 000000000..d3ef460ad --- /dev/null +++ b/internal/testcommand/match_test.go @@ -0,0 +1,59 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testcommand + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMatchesPackageScripts(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"scripts":{"test":"npm run unit","unit":"jest --ci","cycle":"npm run cycle","lint":"eslint ."}}`), 0600)) + for _, tc := range []struct { + command string + want bool + }{ + {"npm test", true}, + {"pnpm run unit", true}, + {"npm run lint", false}, + {"npm run cycle", false}, + {"npm install jest", false}, + {"npx --no-install jest --ci", true}, + } { + t.Run(tc.command, func(t *testing.T) { + require.Equal(t, tc.want, Matches(root, tc.command, "javascript", "jest")) + }) + } +} + +func TestMatchesToxEnvironments(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "pyproject.toml"), []byte(`[tool.tox] +env_list = ["tests"] +[tool.tox.env.tests] +commands = [["python", "-m", "pytest"]] +[tool.tox.env.lint] +commands = [["ruff", "check"]] +`), 0600)) + for _, tc := range []struct { + command string + want bool + }{ + {"tox", true}, + {"tox -e lint", false}, + {"tox --environment=lint,tests", true}, + {"tox -etests", true}, + {"tox -e unknown", true}, + } { + t.Run(tc.command, func(t *testing.T) { + require.Equal(t, tc.want, Matches(root, tc.command, "python", "pytest")) + }) + } +} diff --git a/internal/testdrive/onboard.go b/internal/testdrive/workflow.go similarity index 74% rename from internal/testdrive/onboard.go rename to internal/testdrive/workflow.go index c653b00b5..c8a7ede96 100644 --- a/internal/testdrive/onboard.go +++ b/internal/testdrive/workflow.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" "io" - "maps" "net/url" "os" "slices" @@ -40,53 +39,11 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if !t.lang().coordinated() && options.Format == "verbose" && !options.Apply && options.Resume == "" && options.FactsFile == "" { return t.Run(ctx, output) } - result := onboard.AgentResult{Status: "running", Phase: "discovery", NextAction: "run_command", NextCommand: onboard.OnboardingCommand(name)} + result := onboard.AgentResult{Status: "running", Phase: "discovery", NextAction: "run_command", NextCommand: t.workflowCommand(options)} if len(result.NextCommand) == 0 { result.Status, result.Code, result.Reason, result.NextAction = "needs_input", "transient_executable", onboard.TransientExecutableReason, "supply_input" return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, options.Format)) } - if !options.Apply { - result.NextCommand = onboard.DDTestCommand("testdrive", "--framework", name, "--agent", "--yes") - if t.allConfigurations { - result.NextCommand = append(result.NextCommand, "--all") - } - if override := strings.TrimSpace(settings.GetCommand()); !t.allConfigurations && override != "" { - result.NextCommand = append(result.NextCommand, "--command", override) - } - } - if options.Format == "json" { - result.NextCommand = append(result.NextCommand, "--format", "json") - } else if options.Apply && options.Format == "agent" { - result.NextCommand = append(result.NextCommand, "--agent") - } - - if t.commandTimeout > 0 { - result.NextCommand = append(result.NextCommand, "--timeout", t.commandTimeout.String()) - } - if t.allowReviewedBuilds { - result.NextCommand = append(result.NextCommand, "--allow-reviewed-builds") - } - if t.acceptSubstituteRuntime { - result.NextCommand = append(result.NextCommand, AcceptSubstituteRuntimeFlag) - } - if options.ExplicitVersion { - result.NextCommand = append(result.NextCommand, "--tracer-version", t.tracerVersion) - } - if options.FactsFile != "" { - result.NextCommand = append(result.NextCommand, "--facts", options.FactsFile) - } - if selectedPlatform := settings.GetPlatform(); selectedPlatform != "" { - result.NextCommand = append(result.NextCommand, "--platform", selectedPlatform) - } - if options.Apply && settings.GetCommand() != "" { - result.NextCommand = append(result.NextCommand, "--command", settings.GetCommand()) - } - if options.AllowDirty { - result.NextCommand = append(result.NextCommand, "--allow-dirty") - } - if options.Site != "" { - result.NextCommand = append(result.NextCommand, "--site", options.Site) - } recoveryCommand := append([]string{}, result.NextCommand...) var checkpoint *workflowCheckpoint var factsHash string @@ -917,350 +874,6 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work return onboard.RenderAgentResult(output, result, options.Format) } -// splitTracerOnlyBlockers separates blockers only a tracer change can address -// (with their reasons) from the rest. The summary lines that merely restate a -// tracer-only feature failure are dropped. -func splitTracerOnlyBlockers(validation validationResult) (reportOnly, actionable []string) { - tracerOnly, _ := tracerOnlyFeatures(validation) - for _, check := range validation.Summary.BlockingChecks { - if feature, ok := tracerOnly[check]; ok { - reportOnly = append(reportOnly, check+" — "+feature.Reason) - continue - } - if restatesTracerOnlyFailures(validation, check) { - continue - } - actionable = append(actionable, check) - } - // A configuration's "incomplete" line only restates the other blockers that - // caused it; keep it only when it is the sole explanation. - derived := func(check string) bool { return configurationStatusLine(validation, check) } - if slices.ContainsFunc(validation.Summary.BlockingChecks, func(check string) bool { return !derived(check) }) { - actionable = slices.DeleteFunc(actionable, derived) - } - return reportOnly, actionable -} - -// tracerOnlyFeatures maps each unsatisfied feature's blocker line to the -// feature when only a tracer change can address it, and reports whether any -// other feature failed. -func tracerOnlyFeatures(validation validationResult) (map[string]featureResult, bool) { - tracerOnly := map[string]featureResult{} - others := false - for _, feature := range validation.Features { - if featureSatisfied(validation.Framework, feature) { - continue - } - if feature.tracerOnly() { - tracerOnly[fmt.Sprintf("%s/%s: %s", feature.Project, feature.Name, feature.Status)] = feature - } else { - others = true - } - } - return tracerOnly, others -} - -// restatesTracerOnlyFailures reports whether check only restates feature -// failures that are all tracer-only: the generic feature-coverage line and a -// compatible configuration's "incomplete" line. -func restatesTracerOnlyFailures(validation validationResult, check string) bool { - tracerOnly, others := tracerOnlyFeatures(validation) - return len(tracerOnly) > 0 && !others && featureResultsComplete(validation) && - (check == featureCoverageBlocker || pairedConfigurationBlocker(validation, check)) -} - -// featureResultsComplete reports whether every project has a result for every -// required feature and every paired configuration produced one, so a missing -// result is never hidden behind tracer-only failures. -func featureResultsComplete(validation validationResult) bool { - projects := map[string]map[string]bool{} - for _, feature := range validation.Features { - if projects[feature.Project] == nil { - projects[feature.Project] = map[string]bool{} - } - projects[feature.Project][feature.Name] = true - } - paired := 0 - for _, configuration := range validation.Configurations { - if configuration.PairedExecution { - paired++ - } - } - if len(projects) == 0 || len(projects) < paired { - return false - } - for _, names := range projects { - for _, policy := range featurePolicies(validation.Framework) { - if !names[policy.Name] { - return false - } - } - } - return true -} - -// configurationStatusLine reports whether check is the summary's derived -// "configuration : " line for one of the configurations. -func configurationStatusLine(validation validationResult, check string) bool { - return slices.ContainsFunc(validation.Configurations, func(c configurationResult) bool { - return check == "configuration "+c.Command+": "+c.Status - }) -} - -// pairedConfigurationBlocker reports whether check only says that a compatible, -// paired configuration is incomplete, which its feature results explain. -func pairedConfigurationBlocker(validation validationResult, check string) bool { - return slices.ContainsFunc(validation.Configurations, func(c configurationResult) bool { - return c.PairedExecution && c.Compatibility.Status == "compatible" && check == "configuration "+c.Command+": "+c.Status - }) -} - -func reattachSetupFacts(updated *onboard.ValidationScope, previous onboard.ValidationScope, afterEdit bool) error { - matched := map[string]bool{} - identity := func(entry onboard.TestCommand) string { - // The owned editor may partition instrumentation while preserving the - // original command/setup. Each resulting row retains its own new ID. - entry.CIInstrumentationExcluded = false - entry.Environment = maps.Clone(entry.Environment) - if entry.Environment["NODE_OPTIONS"] == "" { - delete(entry.Environment, "NODE_OPTIONS") - } - if len(entry.Environment) == 0 { - entry.Environment = nil - } - scope := onboard.ValidationScope{Commands: []onboard.TestCommand{entry}} - onboard.AssignConfigurationIDs(&scope) - return scope.Commands[0].ID - } - for i := range updated.Commands { - current := updated.Commands[i] - match := slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return entry.ID == current.ID }) - if match < 0 && afterEdit { - match = slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return identity(entry) == identity(current) }) - if match < 0 && strings.HasPrefix(current.Command, "nyc -- ") { - // The bounded editor may add this exact coverage wrapper. Match an - // existing wrapped command first, before considering its old leaf. - current.Command = strings.TrimPrefix(current.Command, "nyc -- ") - match = slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return identity(entry) == identity(current) }) - } - } - if match < 0 { - continue - } - entry := previous.Commands[match] - matched[entry.ID] = true - updated.Commands[i].Services = entry.Services - updated.Commands[i].PortOverride = entry.PortOverride - - for _, build := range entry.Prerequisites { - if build.FactSource != "" { - pending := slices.IndexFunc(updated.Commands[i].Prerequisites, func(candidate onboard.BuildCommand) bool { - return candidate.Kind == onboard.PreparationNeedsFacts && candidate.Command == build.Command && candidate.Directory == build.Directory - }) - if pending >= 0 { - updated.Commands[i].Prerequisites[pending] = build - } else { - updated.Commands[i].Prerequisites = append(updated.Commands[i].Prerequisites, build) - } - } - } - } - for _, entry := range previous.Commands { - hasFacts := len(entry.Services) > 0 || entry.PortOverride != nil || len(entry.LocalOverrides) > 0 - for _, build := range entry.Prerequisites { - hasFacts = hasFacts || build.FactSource != "" - } - if hasFacts && !matched[entry.ID] { - return fmt.Errorf("accepted facts for %s could not be reattached", entry.ID) - } - } - return nil -} - -func (t *Testdrive) selectWorkflowTracer(ctx context.Context, scope onboard.ValidationScope, explicit bool) (string, error) { - ci := onboard.CheckCIRuntimes(ctx, t.repositoryRoot, t.framework.Name()) - if t.workflowPlan != nil { - t.workflowPlan.CIRuntime = &ci - } - var selected platform.JSSelection - t.workflowReportOnly = nil - for _, entry := range scope.Commands { - // Entries partial apply leaves unedited are reported, never preflighted. - if t.setupStaysReported(entry, true) { - continue - } - // One tracer serves every configuration: later entries check the release - // the first one selected instead of recommending their own. - probe, pinned := t, explicit - if selected.Version != "" { - pinnedDrive := *t - pinnedDrive.tracerVersion = selected.Version - probe, pinned = &pinnedDrive, true - } - var candidate platform.JSSelection - var reportOnly []string - err := workflowPreflight(ctx, func() (err error) { - candidate, reportOnly, err = probe.selectSetupTracerReporting(ctx, onboard.ValidationScope{Commands: []onboard.TestCommand{entry}}, pinned) - return err - }) - if err != nil { - return selected.Version, err - } - t.workflowReportOnly = append(t.workflowReportOnly, reportOnly...) - if candidate.Version == "" { - continue - } - if selected.Version != "" && selected.Version != candidate.Version { - return selected.Version, fmt.Errorf("tracer selection changed across configurations") - } - selected = candidate - if t.workflowPlan != nil { - t.workflowPlan.TracerNode = selected.Node - t.workflowPlan.Tracer = selected.Version - } - } - if selected.Version == "" { - if len(t.workflowReportOnly) > 0 { - return "", noRunnableConfiguration(t.workflowReportOnly) - } - return "", fmt.Errorf("no configurations enabled for instrumentation; explicit CI opt-outs are not validations") - } - return selected.Version, nil -} -func workflowMetadataCapability(metadata *platform.MetadataError) string { - switch metadata.Code { - case "EACCES", "EPERM", "EROFS", "ENOSPC": - return "private temporary storage" - } - return "package metadata" -} -func workflowMetadataRetryable(code string) bool { - switch code { - case "EAI_AGAIN", "ENETUNREACH", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "FETCH_ERROR": - return true - } - return false -} - -// Only idempotent metadata preflight may retry, and only one recognized -// transient transport failure. Tests, auth and permission failures never retry. -func workflowPreflight(ctx context.Context, run func() error) error { - err := run() - var metadata *platform.MetadataError - if errors.As(err, &metadata) && workflowMetadataRetryable(metadata.Code) && ctx.Err() == nil { - return run() - } - return err -} - -func readCurrentOnboardingResult(root string, started time.Time) (validationResult, error) { - var result validationResult - // A run of one command after an --all run writes beside that report. - if data, err := os.ReadFile(validationResult{KeptReport: validationPath("")}.jsonReportPath(root)); err == nil { - if json.Unmarshal(data, &result) == nil && !result.CheckOnly && !result.CompletedAt.Before(started) { - return result, nil - } - result = validationResult{} - } - data, err := os.ReadFile(validationPath(root)) - if err != nil { - return result, err - } - if err := json.Unmarshal(data, &result); err != nil { - return validationResult{}, err - } - if result.CheckOnly || result.CompletedAt.Before(started) { - return validationResult{}, fmt.Errorf("the report does not describe a full execution from this operation; historical or check-only evidence cannot establish completion") - } - return result, nil -} - -// RenderReport is read-only. It cannot certify modified source or a new Node -// runtime, and exposes the same structured result to hosts without an LLM. -func RenderReport(ctx context.Context, output io.Writer, format string) error { - root, err := os.Getwd() - if err != nil { - return renderReportFailure(output, format, err) - } - root = workspace.CanonicalRoot(root) - lock, err := lockReports(root) - if err != nil { - result := onboard.AgentResult{Status: "blocked_environment", Phase: "report", Code: "coordination_unavailable", Reason: "A validation is active or coordination storage is unavailable; wait before rendering saved evidence.", NextAction: "wait"} - if errors.Is(err, workspace.ErrBusy) { - result.Code = "workflow_busy" - } else { - result.NextAction = "request_capability" - } - return errors.Join(err, onboard.RenderAgentResult(output, result, format)) - } - defer func() { _ = lock.Close() }() - validation, err := readPreviousValidation(root) - if err != nil { - return renderReportFailure(output, format, err) - } - if validation == nil { - return renderReportFailure(output, format, fmt.Errorf("no saved validation report")) - } - if result, handled := languageOrUnsupported(validation.Framework).standaloneReport(root, validation); handled { - return onboard.RenderAgentResult(output, result, format) - } - if !supportsValidationFramework(validation.Framework) || !languageOrUnsupported(validation.Framework).coordinated() { - result := onboard.AgentResult{Status: "unsupported", Phase: "report", Code: "framework_unsupported", Reason: "Structured onboarding reports are not available for this framework. Inspect the saved standalone HTML/JSON as historical evidence, or rerun standalone testdrive.", NextAction: "report_result"} - return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) - } - if validation.CheckOnly && validation.Retained != nil && validation.Retained.Result != nil { - validation = validation.Retained.Result - } - result := onboard.AgentResult{Status: "validation_failed", Phase: "report", NextAction: "report_result", JSONReport: validationPath(root)} - // A focused run in a package below the root bound its evidence to the - // package: check the inputs it validated. - runRoot := reportRunRoot(root, *validation) - current, err := workflowFingerprint(ctx, runRoot, validation.Framework) - if err != nil { - return renderReportFailure(output, format, err) - } - receiptValid := verifyWorkflowReceipt(root, *validation) == nil - if validation.CheckOnly || validation.Workflow == nil || validation.Workflow.Fingerprint != current || !languageOrUnsupported(validation.Framework).reportRuntimeCurrent(runRoot, *validation.Workflow) || !receiptValid { - result.Code = "stale_report" - plain := !validation.CheckOnly && validation.Workflow == nil - result.NextAction = "run_command" - result.NextCommand = onboard.DDTestCommand("testdrive", "--framework", validation.Framework, "--all", "--agent", "--yes") - if receiptValid && validation.Workflow != nil && len(validation.Workflow.ValidationCommand) > 0 { - result.NextCommand = append([]string{}, validation.Workflow.ValidationCommand...) - } - if len(result.NextCommand) == 0 { - result.NextAction = "supply_input" - result.Code = "transient_executable" - result.Reason = onboard.TransientExecutableReason - return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) - } - if format == "json" && !slices.Contains(result.NextCommand, "json") { - result.NextCommand = append(result.NextCommand, "--format", "json") - } - if receiptValid && validation.Selection != nil && validation.Selection.Version != "" { - result.NextCommand = append(result.NextCommand, "--tracer-version", validation.Selection.Version) - } - result.Reason = "Saved evidence does not certify the current source/runtime. Run full testdrive; do not report historical success." - if plain { - // Plain runs replace the report without the workflow's checks or binding. - result.Code = "plain_report" - result.Reason = "The saved report comes from a plain testdrive run, which the agent workflow neither produced nor certifies. Run next_command to validate through the workflow." - } - return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) - } - result.Fingerprint = current - result.Checks = validation.Workflow.Checks - result.FinalResponse = validation.Summary.FinalResponse - if validation.HTMLReportCurrent { - result.HTMLReport = htmlReportPath(root) - } - result.Status = workflowVerdict(*validation) - if result.Status == "validation_failed" { - return errors.Join(fmt.Errorf("saved validation remains incomplete"), onboard.RenderAgentResult(output, result, format)) - } - return onboard.RenderAgentResult(output, result, format) -} - // Rejected options, facts or resume IDs describe this invocation, not the // repository, so they leave an earlier report and its binding untouched. func workflowInputFailure(code string) bool { @@ -1271,27 +884,6 @@ func workflowInputFailure(code string) bool { return false } -// A stopped attempt invalidates the current handoff while keeping one historical -// execution. Never overwrite a different framework or an unreadable report. -func invalidatePriorWorkflow(root, framework string, state onboard.AgentResult) { - previous, err := readPreviousValidation(root) - if err != nil || previous == nil || previous.Framework != framework { - return - } - result := validationResult{Session: workspace.PlanSession().ID(), Framework: framework, Error: state.Code + ": " + state.Reason, StoppedBeforeExecution: true, - Compatibility: verdict{Status: "not exercised", Reason: "This workflow attempt stopped before a current full execution."}, - Workflow: &WorkflowEvidence{Fingerprint: state.Fingerprint, Node: currentNodeVersion(), Checks: []onboard.WorkflowCheck{{Name: state.Code, Status: "failed", Reason: state.Reason}}}} - _ = finishValidation(io.Discard, root, result) // A successfully written INCOMPLETE report also returns an error. -} - -func renderReportFailure(output io.Writer, format string, err error) error { - result := onboard.AgentResult{Status: "needs_input", Phase: "report", Code: "report_unavailable", Reason: reportText(err.Error()), NextAction: "report_result"} - if errors.Is(err, errReportUnreadable) { - result.Code, result.Reason = "report_unreadable", errReportUnreadable.Error() - } - return errors.Join(err, onboard.RenderAgentResult(output, result, format)) -} - // withWorkflowState carries the workflow's scope, plan, run and selected site // from current into t, a drive prepared again for another tracer release. func (t *Testdrive) withWorkflowState(current *Testdrive) *Testdrive { @@ -1307,29 +899,6 @@ func (t *Testdrive) workflowNodeVersion() string { return t.nodeVersion() } -func (t *Testdrive) workflowValidationResume(executable, name, selected string, options WorkflowOptions) []string { - command := []string{executable, "testdrive", "--framework", name, "--all", "--agent", "--tracer-version", selected, "--yes"} - if options.Format == "json" { - command = append(command, "--format", "json") - } - for _, flag := range []struct{ name, value string }{{"ci-provider", onboard.ActiveCIProvider()}, {"facts", options.FactsFile}, {"site", options.Site}, {"platform", settings.GetPlatform()}, {"command", settings.GetCommand()}} { - if flag.value != "" { - command = append(command, "--"+flag.name, flag.value) - } - } - command = append(command, onboard.CIStatementArguments()...) - if t.commandTimeout > 0 { - command = append(command, "--timeout", t.commandTimeout.String()) - } - if t.allowReviewedBuilds { - command = append(command, "--allow-reviewed-builds") - } - if t.acceptSubstituteRuntime { - command = append(command, AcceptSubstituteRuntimeFlag) - } - return command -} - // scopeCheck records that ready configurations proceeded around unresolved CI. // The summary lists the entries once, as scope blockers, not again as a check. func scopeCheck(unresolved []string) onboard.WorkflowCheck { diff --git a/internal/testdrive/workflow_commands.go b/internal/testdrive/workflow_commands.go new file mode 100644 index 000000000..48344e11a --- /dev/null +++ b/internal/testdrive/workflow_commands.go @@ -0,0 +1,86 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "strings" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/settings" +) + +// workflowCommand preserves the options needed to restart this operation. +func (t *Testdrive) workflowCommand(options WorkflowOptions) []string { + name := t.framework.Name() + command := onboard.OnboardingCommand(name) + if len(command) == 0 { + return nil + } + if !options.Apply { + command = onboard.DDTestCommand("testdrive", "--framework", name, "--agent", "--yes") + if t.allConfigurations { + command = append(command, "--all") + } + if override := strings.TrimSpace(settings.GetCommand()); !t.allConfigurations && override != "" { + command = append(command, "--command", override) + } + } + if options.Format == "json" { + command = append(command, "--format", "json") + } else if options.Apply && options.Format == "agent" { + command = append(command, "--agent") + } + + command = append(command, t.workflowExecutionArguments()...) + if options.ExplicitVersion { + command = append(command, "--tracer-version", t.tracerVersion) + } + if options.FactsFile != "" { + command = append(command, "--facts", options.FactsFile) + } + if selectedPlatform := settings.GetPlatform(); selectedPlatform != "" { + command = append(command, "--platform", selectedPlatform) + } + if options.Apply && settings.GetCommand() != "" { + command = append(command, "--command", settings.GetCommand()) + } + if options.AllowDirty { + command = append(command, "--allow-dirty") + } + if options.Site != "" { + command = append(command, "--site", options.Site) + } + return command +} + +func (t *Testdrive) workflowValidationResume(executable, name, selected string, options WorkflowOptions) []string { + command := []string{executable, "testdrive", "--framework", name, "--all", "--agent", "--tracer-version", selected, "--yes"} + if options.Format == "json" { + command = append(command, "--format", "json") + } + for _, flag := range []struct{ name, value string }{{"ci-provider", onboard.ActiveCIProvider()}, {"facts", options.FactsFile}, {"site", options.Site}, {"platform", settings.GetPlatform()}, {"command", settings.GetCommand()}} { + if flag.value != "" { + command = append(command, "--"+flag.name, flag.value) + } + } + command = append(command, onboard.CIStatementArguments()...) + return append(command, t.workflowExecutionArguments()...) +} + +// workflowExecutionArguments keeps execution policy identical on restart and resume. +func (t *Testdrive) workflowExecutionArguments() []string { + var arguments []string + if t.commandTimeout > 0 { + arguments = append(arguments, "--timeout", t.commandTimeout.String()) + } + if t.allowReviewedBuilds { + arguments = append(arguments, "--allow-reviewed-builds") + } + if t.acceptSubstituteRuntime { + arguments = append(arguments, AcceptSubstituteRuntimeFlag) + } + return arguments +} diff --git a/internal/testdrive/workflow_inputs.go b/internal/testdrive/workflow_inputs.go index a1a7791f3..48f490567 100644 --- a/internal/testdrive/workflow_inputs.go +++ b/internal/testdrive/workflow_inputs.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "io/fs" + "maps" "os" "os/exec" "path/filepath" @@ -517,3 +518,65 @@ func excludedFingerprintPath(name string) bool { name = filepath.ToSlash(name) return name == ".testoptimization" || strings.HasPrefix(name, ".testoptimization/") } + +func reattachSetupFacts(updated *onboard.ValidationScope, previous onboard.ValidationScope, afterEdit bool) error { + matched := map[string]bool{} + identity := func(entry onboard.TestCommand) string { + // The owned editor may partition instrumentation while preserving the + // original command/setup. Each resulting row retains its own new ID. + entry.CIInstrumentationExcluded = false + entry.Environment = maps.Clone(entry.Environment) + if entry.Environment["NODE_OPTIONS"] == "" { + delete(entry.Environment, "NODE_OPTIONS") + } + if len(entry.Environment) == 0 { + entry.Environment = nil + } + scope := onboard.ValidationScope{Commands: []onboard.TestCommand{entry}} + onboard.AssignConfigurationIDs(&scope) + return scope.Commands[0].ID + } + for i := range updated.Commands { + current := updated.Commands[i] + match := slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return entry.ID == current.ID }) + if match < 0 && afterEdit { + match = slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return identity(entry) == identity(current) }) + if match < 0 && strings.HasPrefix(current.Command, "nyc -- ") { + // The bounded editor may add this exact coverage wrapper. Match an + // existing wrapped command first, before considering its old leaf. + current.Command = strings.TrimPrefix(current.Command, "nyc -- ") + match = slices.IndexFunc(previous.Commands, func(entry onboard.TestCommand) bool { return identity(entry) == identity(current) }) + } + } + if match < 0 { + continue + } + entry := previous.Commands[match] + matched[entry.ID] = true + updated.Commands[i].Services = entry.Services + updated.Commands[i].PortOverride = entry.PortOverride + + for _, build := range entry.Prerequisites { + if build.FactSource != "" { + pending := slices.IndexFunc(updated.Commands[i].Prerequisites, func(candidate onboard.BuildCommand) bool { + return candidate.Kind == onboard.PreparationNeedsFacts && candidate.Command == build.Command && candidate.Directory == build.Directory + }) + if pending >= 0 { + updated.Commands[i].Prerequisites[pending] = build + } else { + updated.Commands[i].Prerequisites = append(updated.Commands[i].Prerequisites, build) + } + } + } + } + for _, entry := range previous.Commands { + hasFacts := len(entry.Services) > 0 || entry.PortOverride != nil || len(entry.LocalOverrides) > 0 + for _, build := range entry.Prerequisites { + hasFacts = hasFacts || build.FactSource != "" + } + if hasFacts && !matched[entry.ID] { + return fmt.Errorf("accepted facts for %s could not be reattached", entry.ID) + } + } + return nil +} diff --git a/internal/testdrive/workflow_report.go b/internal/testdrive/workflow_report.go new file mode 100644 index 000000000..8e100893b --- /dev/null +++ b/internal/testdrive/workflow_report.go @@ -0,0 +1,247 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "slices" + "time" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testdrive/workspace" +) + +// splitTracerOnlyBlockers separates blockers only a tracer change can address +// (with their reasons) from the rest. The summary lines that merely restate a +// tracer-only feature failure are dropped. +func splitTracerOnlyBlockers(validation validationResult) (reportOnly, actionable []string) { + tracerOnly, _ := tracerOnlyFeatures(validation) + for _, check := range validation.Summary.BlockingChecks { + if feature, ok := tracerOnly[check]; ok { + reportOnly = append(reportOnly, check+" — "+feature.Reason) + continue + } + if restatesTracerOnlyFailures(validation, check) { + continue + } + actionable = append(actionable, check) + } + // A configuration's "incomplete" line only restates the other blockers that + // caused it; keep it only when it is the sole explanation. + derived := func(check string) bool { return configurationStatusLine(validation, check) } + if slices.ContainsFunc(validation.Summary.BlockingChecks, func(check string) bool { return !derived(check) }) { + actionable = slices.DeleteFunc(actionable, derived) + } + return reportOnly, actionable +} + +// tracerOnlyFeatures maps each unsatisfied feature's blocker line to the +// feature when only a tracer change can address it, and reports whether any +// other feature failed. +func tracerOnlyFeatures(validation validationResult) (map[string]featureResult, bool) { + tracerOnly := map[string]featureResult{} + others := false + for _, feature := range validation.Features { + if featureSatisfied(validation.Framework, feature) { + continue + } + if feature.tracerOnly() { + tracerOnly[fmt.Sprintf("%s/%s: %s", feature.Project, feature.Name, feature.Status)] = feature + } else { + others = true + } + } + return tracerOnly, others +} + +// restatesTracerOnlyFailures reports whether check only restates feature +// failures that are all tracer-only: the generic feature-coverage line and a +// compatible configuration's "incomplete" line. +func restatesTracerOnlyFailures(validation validationResult, check string) bool { + tracerOnly, others := tracerOnlyFeatures(validation) + return len(tracerOnly) > 0 && !others && featureResultsComplete(validation) && + (check == featureCoverageBlocker || pairedConfigurationBlocker(validation, check)) +} + +// featureResultsComplete reports whether every project has a result for every +// required feature and every paired configuration produced one, so a missing +// result is never hidden behind tracer-only failures. +func featureResultsComplete(validation validationResult) bool { + projects := map[string]map[string]bool{} + for _, feature := range validation.Features { + if projects[feature.Project] == nil { + projects[feature.Project] = map[string]bool{} + } + projects[feature.Project][feature.Name] = true + } + paired := 0 + for _, configuration := range validation.Configurations { + if configuration.PairedExecution { + paired++ + } + } + if len(projects) == 0 || len(projects) < paired { + return false + } + for _, names := range projects { + for _, policy := range featurePolicies(validation.Framework) { + if !names[policy.Name] { + return false + } + } + } + return true +} + +// configurationStatusLine reports whether check is the summary's derived +// "configuration : " line for one of the configurations. +func configurationStatusLine(validation validationResult, check string) bool { + return slices.ContainsFunc(validation.Configurations, func(c configurationResult) bool { + return check == "configuration "+c.Command+": "+c.Status + }) +} + +// pairedConfigurationBlocker reports whether check only says that a compatible, +// paired configuration is incomplete, which its feature results explain. +func pairedConfigurationBlocker(validation validationResult, check string) bool { + return slices.ContainsFunc(validation.Configurations, func(c configurationResult) bool { + return c.PairedExecution && c.Compatibility.Status == "compatible" && check == "configuration "+c.Command+": "+c.Status + }) +} + +func readCurrentOnboardingResult(root string, started time.Time) (validationResult, error) { + var result validationResult + // A run of one command after an --all run writes beside that report. + if data, err := os.ReadFile(validationResult{KeptReport: validationPath("")}.jsonReportPath(root)); err == nil { + if json.Unmarshal(data, &result) == nil && !result.CheckOnly && !result.CompletedAt.Before(started) { + return result, nil + } + result = validationResult{} + } + data, err := os.ReadFile(validationPath(root)) + if err != nil { + return result, err + } + if err := json.Unmarshal(data, &result); err != nil { + return validationResult{}, err + } + if result.CheckOnly || result.CompletedAt.Before(started) { + return validationResult{}, fmt.Errorf("the report does not describe a full execution from this operation; historical or check-only evidence cannot establish completion") + } + return result, nil +} + +// RenderReport is read-only. It cannot certify modified source or a new Node +// runtime, and exposes the same structured result to hosts without an LLM. +func RenderReport(ctx context.Context, output io.Writer, format string) error { + root, err := os.Getwd() + if err != nil { + return renderReportFailure(output, format, err) + } + root = workspace.CanonicalRoot(root) + lock, err := lockReports(root) + if err != nil { + result := onboard.AgentResult{Status: "blocked_environment", Phase: "report", Code: "coordination_unavailable", Reason: "A validation is active or coordination storage is unavailable; wait before rendering saved evidence.", NextAction: "wait"} + if errors.Is(err, workspace.ErrBusy) { + result.Code = "workflow_busy" + } else { + result.NextAction = "request_capability" + } + return errors.Join(err, onboard.RenderAgentResult(output, result, format)) + } + defer func() { _ = lock.Close() }() + validation, err := readPreviousValidation(root) + if err != nil { + return renderReportFailure(output, format, err) + } + if validation == nil { + return renderReportFailure(output, format, fmt.Errorf("no saved validation report")) + } + if result, handled := languageOrUnsupported(validation.Framework).standaloneReport(root, validation); handled { + return onboard.RenderAgentResult(output, result, format) + } + if !supportsValidationFramework(validation.Framework) || !languageOrUnsupported(validation.Framework).coordinated() { + result := onboard.AgentResult{Status: "unsupported", Phase: "report", Code: "framework_unsupported", Reason: "Structured onboarding reports are not available for this framework. Inspect the saved standalone HTML/JSON as historical evidence, or rerun standalone testdrive.", NextAction: "report_result"} + return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) + } + if validation.CheckOnly && validation.Retained != nil && validation.Retained.Result != nil { + validation = validation.Retained.Result + } + result := onboard.AgentResult{Status: "validation_failed", Phase: "report", NextAction: "report_result", JSONReport: validationPath(root)} + // A focused run in a package below the root bound its evidence to the + // package: check the inputs it validated. + runRoot := reportRunRoot(root, *validation) + current, err := workflowFingerprint(ctx, runRoot, validation.Framework) + if err != nil { + return renderReportFailure(output, format, err) + } + receiptValid := verifyWorkflowReceipt(root, *validation) == nil + if validation.CheckOnly || validation.Workflow == nil || validation.Workflow.Fingerprint != current || !languageOrUnsupported(validation.Framework).reportRuntimeCurrent(runRoot, *validation.Workflow) || !receiptValid { + result.Code = "stale_report" + plain := !validation.CheckOnly && validation.Workflow == nil + result.NextAction = "run_command" + result.NextCommand = onboard.DDTestCommand("testdrive", "--framework", validation.Framework, "--all", "--agent", "--yes") + if receiptValid && validation.Workflow != nil && len(validation.Workflow.ValidationCommand) > 0 { + result.NextCommand = append([]string{}, validation.Workflow.ValidationCommand...) + } + if len(result.NextCommand) == 0 { + result.NextAction = "supply_input" + result.Code = "transient_executable" + result.Reason = onboard.TransientExecutableReason + return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) + } + if format == "json" && !slices.Contains(result.NextCommand, "json") { + result.NextCommand = append(result.NextCommand, "--format", "json") + } + if receiptValid && validation.Selection != nil && validation.Selection.Version != "" { + result.NextCommand = append(result.NextCommand, "--tracer-version", validation.Selection.Version) + } + result.Reason = "Saved evidence does not certify the current source/runtime. Run full testdrive; do not report historical success." + if plain { + // Plain runs replace the report without the workflow's checks or binding. + result.Code = "plain_report" + result.Reason = "The saved report comes from a plain testdrive run, which the agent workflow neither produced nor certifies. Run next_command to validate through the workflow." + } + return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, format)) + } + result.Fingerprint = current + result.Checks = validation.Workflow.Checks + result.FinalResponse = validation.Summary.FinalResponse + if validation.HTMLReportCurrent { + result.HTMLReport = htmlReportPath(root) + } + result.Status = workflowVerdict(*validation) + if result.Status == "validation_failed" { + return errors.Join(fmt.Errorf("saved validation remains incomplete"), onboard.RenderAgentResult(output, result, format)) + } + return onboard.RenderAgentResult(output, result, format) +} + +// A stopped attempt invalidates the current handoff while keeping one historical +// execution. Never overwrite a different framework or an unreadable report. +func invalidatePriorWorkflow(root, framework string, state onboard.AgentResult) { + previous, err := readPreviousValidation(root) + if err != nil || previous == nil || previous.Framework != framework { + return + } + result := validationResult{Session: workspace.PlanSession().ID(), Framework: framework, Error: state.Code + ": " + state.Reason, StoppedBeforeExecution: true, + Compatibility: verdict{Status: "not exercised", Reason: "This workflow attempt stopped before a current full execution."}, + Workflow: &WorkflowEvidence{Fingerprint: state.Fingerprint, Node: currentNodeVersion(), Checks: []onboard.WorkflowCheck{{Name: state.Code, Status: "failed", Reason: state.Reason}}}} + _ = finishValidation(io.Discard, root, result) // A successfully written INCOMPLETE report also returns an error. +} + +func renderReportFailure(output io.Writer, format string, err error) error { + result := onboard.AgentResult{Status: "needs_input", Phase: "report", Code: "report_unavailable", Reason: reportText(err.Error()), NextAction: "report_result"} + if errors.Is(err, errReportUnreadable) { + result.Code, result.Reason = "report_unreadable", errReportUnreadable.Error() + } + return errors.Join(err, onboard.RenderAgentResult(output, result, format)) +} diff --git a/internal/testdrive/workflow_tracer.go b/internal/testdrive/workflow_tracer.go new file mode 100644 index 000000000..a45950263 --- /dev/null +++ b/internal/testdrive/workflow_tracer.go @@ -0,0 +1,91 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "errors" + "fmt" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" +) + +func (t *Testdrive) selectWorkflowTracer(ctx context.Context, scope onboard.ValidationScope, explicit bool) (string, error) { + ci := onboard.CheckCIRuntimes(ctx, t.repositoryRoot, t.framework.Name()) + if t.workflowPlan != nil { + t.workflowPlan.CIRuntime = &ci + } + var selected platform.JSSelection + t.workflowReportOnly = nil + for _, entry := range scope.Commands { + // Entries partial apply leaves unedited are reported, never preflighted. + if t.setupStaysReported(entry, true) { + continue + } + // One tracer serves every configuration: later entries check the release + // the first one selected instead of recommending their own. + probe, pinned := t, explicit + if selected.Version != "" { + pinnedDrive := *t + pinnedDrive.tracerVersion = selected.Version + probe, pinned = &pinnedDrive, true + } + var candidate platform.JSSelection + var reportOnly []string + err := workflowPreflight(ctx, func() (err error) { + candidate, reportOnly, err = probe.selectSetupTracerReporting(ctx, onboard.ValidationScope{Commands: []onboard.TestCommand{entry}}, pinned) + return err + }) + if err != nil { + return selected.Version, err + } + t.workflowReportOnly = append(t.workflowReportOnly, reportOnly...) + if candidate.Version == "" { + continue + } + if selected.Version != "" && selected.Version != candidate.Version { + return selected.Version, fmt.Errorf("tracer selection changed across configurations") + } + selected = candidate + if t.workflowPlan != nil { + t.workflowPlan.TracerNode = selected.Node + t.workflowPlan.Tracer = selected.Version + } + } + if selected.Version == "" { + if len(t.workflowReportOnly) > 0 { + return "", noRunnableConfiguration(t.workflowReportOnly) + } + return "", fmt.Errorf("no configurations enabled for instrumentation; explicit CI opt-outs are not validations") + } + return selected.Version, nil +} +func workflowMetadataCapability(metadata *platform.MetadataError) string { + switch metadata.Code { + case "EACCES", "EPERM", "EROFS", "ENOSPC": + return "private temporary storage" + } + return "package metadata" +} +func workflowMetadataRetryable(code string) bool { + switch code { + case "EAI_AGAIN", "ENETUNREACH", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "FETCH_ERROR": + return true + } + return false +} + +// Only idempotent metadata preflight may retry, and only one recognized +// transient transport failure. Tests, auth and permission failures never retry. +func workflowPreflight(ctx context.Context, run func() error) error { + err := run() + var metadata *platform.MetadataError + if errors.As(err, &metadata) && workflowMetadataRetryable(metadata.Code) && ctx.Err() == nil { + return run() + } + return err +} From 633c47997f03c88d7605628b8e9c6f1f34947725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 16:35:40 +0100 Subject: [PATCH 2/5] Preserve manual guidance for GitHub environment expressions --- internal/onboard/README.md | 3 +- internal/onboard/ci_provider_github.go | 75 +++++++++++++++------ internal/onboard/ci_provider_github_test.go | 47 +++++++++++++ 3 files changed, 103 insertions(+), 22 deletions(-) diff --git a/internal/onboard/README.md b/internal/onboard/README.md index 9182695a1..71cce28a8 100644 --- a/internal/onboard/README.md +++ b/internal/onboard/README.md @@ -32,7 +32,8 @@ Providers parse their own syntax into `ciWorkflow`, `runtimeJob` and language decisions stay behind `discoveryLanguage`. Providers normally use `findTestWorkflows`' shared classification. GitHub manual instructions preserve their conservative literal-command inspection and use the same GitHub file -loader as full discovery; full discovery additionally expands composite actions. +loader as full discovery. Instructions decode only the fields they inspect; +full discovery decodes the complete workflow model and expands composite actions. `internal/environment` detects the running CI environment and supplies runtime tags and runner configuration. Repository CI inspection belongs here, in diff --git a/internal/onboard/ci_provider_github.go b/internal/onboard/ci_provider_github.go index 074f5721c..979c2b9c3 100644 --- a/internal/onboard/ci_provider_github.go +++ b/internal/onboard/ci_provider_github.go @@ -61,28 +61,37 @@ func (g gitHubActions) configurationFiles(root string) ([]string, error) { return paths, err } -func (g gitHubActions) readWorkflows(root string) ([]ciWorkflow, error) { - workflows, err := g.readWorkflowFiles(root) +func (gitHubActions) readWorkflows(root string) ([]ciWorkflow, error) { + files, err := readGitHubWorkflowFiles[ciWorkflow](root) if err != nil { return nil, err } - for _, workflow := range workflows { + var workflows []ciWorkflow + for _, file := range files { + workflow := file.workflow + workflow.Path = file.path for jobName, job := range workflow.Jobs { job.Steps = expandCompositeSteps(root, workflow.Path, job.Steps) workflow.Jobs[jobName] = job } + workflows = append(workflows, workflow) } return workflows, nil } -// readWorkflowFiles is the single GitHub workflow loader. Manual instructions -// inspect literal steps; discovery expands composite actions after loading. -func (g gitHubActions) readWorkflowFiles(root string) ([]ciWorkflow, error) { - names, err := g.workflowEntries(root) +type gitHubWorkflowFile[T any] struct { + path string + workflow T +} + +// readGitHubWorkflowFiles shares file loading and parse errors while letting +// manual instructions decode fewer fields than full discovery and validation. +func readGitHubWorkflowFiles[T any](root string) ([]gitHubWorkflowFile[T], error) { + names, err := (gitHubActions{}).workflowEntries(root) if err != nil { return nil, err } - var workflows []ciWorkflow + var workflows []gitHubWorkflowFile[T] directory := filepath.Join(root, ".github", "workflows") for _, name := range names { path := filepath.Join(directory, name) @@ -90,28 +99,52 @@ func (g gitHubActions) readWorkflowFiles(root string) ([]ciWorkflow, error) { if err != nil { return nil, fmt.Errorf("read %s: %w", path, err) } - var workflow ciWorkflow + var workflow T if err := yaml.Unmarshal(contents, &workflow); err != nil { return nil, fmt.Errorf("parse %s: %w", path, err) } - workflow.Path = ".github/workflows/" + name - workflows = append(workflows, workflow) + workflows = append(workflows, gitHubWorkflowFile[T]{path: ".github/workflows/" + name, workflow: workflow}) } return workflows, nil } -func (g gitHubActions) findTestWorkflows(root, language, framework string) ([]string, []string, error) { - workflows, err := g.readWorkflowFiles(root) +// Manual instructions inspect only literal commands, directories and setup +// languages. Other fields, including expression-valued environments, must not +// prevent guidance; full validation still decodes the complete ciWorkflow. +type gitHubInstructionDefaults struct { + Run struct { + WorkingDirectory string `yaml:"working-directory"` + } `yaml:"run"` +} + +type gitHubInstructionWorkflow struct { + Defaults gitHubInstructionDefaults `yaml:"defaults"` + Jobs map[string]struct { + Defaults gitHubInstructionDefaults `yaml:"defaults"` + Steps []struct { + Run string `yaml:"run"` + Uses string `yaml:"uses"` + If string `yaml:"if"` + WorkingDirectory string `yaml:"working-directory"` + With struct { + Languages string `yaml:"languages"` + } `yaml:"with"` + } `yaml:"steps"` + } `yaml:"jobs"` +} + +func (gitHubActions) findTestWorkflows(root, language, framework string) ([]string, []string, error) { + workflows, err := readGitHubWorkflowFiles[gitHubInstructionWorkflow](root) if err != nil { return nil, nil, err } var found, configured []string - for _, workflow := range workflows { - tests, setup := inspectLiteralTestJobs(root, workflow, language, framework) + for _, file := range workflows { + tests, setup := inspectLiteralTestJobs(root, file.workflow, language, framework) if tests { - found = append(found, workflow.Path) + found = append(found, file.path) if setup { - configured = append(configured, workflow.Path) + configured = append(configured, file.path) } } } @@ -120,7 +153,7 @@ func (g gitHubActions) findTestWorkflows(root, language, framework string) ([]st // inspectLiteralTestJobs preserves the conservative action and command checks // used by manual onboarding. Full validation uses resolved commands instead. -func inspectLiteralTestJobs(root string, workflow ciWorkflow, language, name string) (foundTests, configured bool) { +func inspectLiteralTestJobs(root string, workflow gitHubInstructionWorkflow, language, name string) (foundTests, configured bool) { actionLanguage := setupActionLanguage(language) for _, job := range workflow.Jobs { jobConfigured := false @@ -146,9 +179,9 @@ func inspectLiteralTestJobs(root string, workflow ciWorkflow, language, name str } } action, _, hasVersion := strings.Cut(strings.ToLower(step.Uses), "@") - if action == githubAction && hasVersion && (condition == "" || condition == "true" || condition == "success()") && !strings.Contains(step.With["languages"], "${{") { - languages := strings.Fields(step.With["languages"]) - jobConfigured = jobConfigured || step.With["languages"] == "all" || slices.Contains(languages, actionLanguage) + if action == githubAction && hasVersion && (condition == "" || condition == "true" || condition == "success()") && !strings.Contains(step.With.Languages, "${{") { + languages := strings.Fields(step.With.Languages) + jobConfigured = jobConfigured || step.With.Languages == "all" || slices.Contains(languages, actionLanguage) } } } diff --git a/internal/onboard/ci_provider_github_test.go b/internal/onboard/ci_provider_github_test.go index a9dff8d8f..c0b34b176 100644 --- a/internal/onboard/ci_provider_github_test.go +++ b/internal/onboard/ci_provider_github_test.go @@ -6,14 +6,61 @@ package onboard import ( + "bytes" "fmt" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" ) +func TestGitHubInstructionsIgnoreExpressionEnvironments(t *testing.T) { + for _, scope := range []string{"workflow", "job", "step"} { + for _, configured := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/configured=%t", scope, configured), func(t *testing.T) { + workflow := `jobs: + tests: + runs-on: ubuntu-latest + steps: + - run: npm test +` + const environment = "env: ${{ fromJSON(vars.TEST_ENV) }}\n" + switch scope { + case "workflow": + workflow = environment + workflow + case "job": + workflow = strings.Replace(workflow, " runs-on:", " "+environment+" runs-on:", 1) + case "step": + workflow += " " + environment + } + if configured { + workflow = strings.Replace(workflow, " steps:\n", " steps:\n - uses: datadog/test-visibility-github-action@v3\n with: {languages: js}\n", 1) + } + root := newJestRepository(t, workflow) + useCIProvider(t, "github") + t.Chdir(root) + + var output bytes.Buffer + require.NoError(t, Run(&output)) + require.Contains(t, output.String(), "DDTest found javascript, jest, and GitHub Actions.") + require.Contains(t, output.String(), ".github/workflows/test.yml") + require.Equal(t, configured, strings.Contains(output.String(), "already appears in every detected test workflow")) + if !configured { + require.Contains(t, output.String(), "languages: js") + } + + // Manual guidance must not make unresolved environments acceptable + // to full discovery and validation. + _, err := (gitHubActions{}).readWorkflows(root) + require.ErrorContains(t, err, "cannot unmarshal") + require.ErrorContains(t, err, "into map[string]string") + }) + } + } +} + func TestGitHubFindTestWorkflowsWorkingDirectories(t *testing.T) { for _, tc := range []struct { name, workflow, job, step string From a2cbbccfaf2c7bb7a98a35e08d6ed3eb2b97029a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 19:22:07 +0100 Subject: [PATCH 3/5] Separate workflow state and shared CI decisions --- internal/onboard/README.md | 14 + internal/onboard/agent.go | 262 +----------- internal/onboard/agent_result.go | 172 ++++++++ internal/onboard/commands.go | 101 +++++ internal/onboard/execution_plan.go | 15 + internal/testdrive/standalone_workflow.go | 30 +- internal/testdrive/workflow.go | 495 +++------------------- internal/testdrive/workflow_attempt.go | 157 +++++++ internal/testdrive/workflow_ci.go | 52 +++ internal/testdrive/workflow_ci_test.go | 75 ++++ internal/testdrive/workflow_completion.go | 148 +++++++ internal/testdrive/workflow_setup.go | 173 ++++++++ 12 files changed, 966 insertions(+), 728 deletions(-) create mode 100644 internal/onboard/agent_result.go create mode 100644 internal/onboard/commands.go create mode 100644 internal/testdrive/workflow_attempt.go create mode 100644 internal/testdrive/workflow_ci.go create mode 100644 internal/testdrive/workflow_ci_test.go create mode 100644 internal/testdrive/workflow_completion.go create mode 100644 internal/testdrive/workflow_setup.go diff --git a/internal/onboard/README.md b/internal/onboard/README.md index 71cce28a8..0a8f6b8f2 100644 --- a/internal/onboard/README.md +++ b/internal/onboard/README.md @@ -7,6 +7,10 @@ Start at the boundary that owns the behavior you need to change: | --- | --- | | CLI flags, consent and output mode | `internal/cmd/onboard.go`, `internal/cmd/testdrive.go` | | Shared CI flags and error rendering | `internal/cmd/ci_provider.go` (`ciOptions`) | +| Onboarding detection and setup handoffs | `internal/onboard/agent.go` | +| Shared agent protocol and rendering | `internal/onboard/agent_result.go` | +| Persistent executable lookup and follow-up commands | `internal/onboard/commands.go` | +| Execution plans and setup-facts schema | `internal/onboard/execution_plan.go` | | CI provider selection and follow-up arguments | `internal/onboard/ci_selection.go` | | Provider contract and registrations | `internal/onboard/ci_provider.go`, `ci_providers.go` | | Workflow model, loading and shared test/setup classification | `internal/onboard/ci_workflows.go` | @@ -19,6 +23,10 @@ Start at the boundary that owns the behavior you need to change: | Setup plans and bounded source edits | `internal/onboard/execution_plan.go`, `apply*.go` | | Validation preparation and execution | `internal/testdrive/testdrive.go` | | Coordinated setup and validation | `internal/testdrive/workflow.go` | +| Attempt state, failure policy and retry handoffs | `internal/testdrive/workflow_attempt.go` | +| Shared CI questions and pinned tracer selection | `internal/testdrive/workflow_ci.go` | +| Checkpoint-bound facts and setup failure recovery | `internal/testdrive/workflow_setup.go` | +| Verified report completion and blocker handoffs | `internal/testdrive/workflow_completion.go` | | Restart and resume commands | `internal/testdrive/workflow_commands.go` | | Workflow tracer selection and metadata preflight | `internal/testdrive/workflow_tracer.go` | | Saved workflow reports and blocker classification | `internal/testdrive/workflow_report.go` | @@ -39,6 +47,12 @@ full discovery decodes the complete workflow model and expands composite actions tags and runner configuration. Repository CI inspection belongs here, in `internal/onboard`, and is shared with testdrive. +`workflowAttempt` owns a single invocation's result, checkpoint and recovery +state. The coordinator sequences discovery, setup, execution and verification; +the attempt handles failure and completion handoffs. Standalone validation +shares the attempt's CI decisions and failure policy while keeping its own +execution and verdict rules. + After changing these paths, run `make test` and `make lint`. The end-to-end Codex catalog is described in [`skills/ddtest-onboarding-eval/SKILL.md`](../../skills/ddtest-onboarding-eval/SKILL.md). diff --git a/internal/onboard/agent.go b/internal/onboard/agent.go index 9442dc5f3..140a33787 100644 --- a/internal/onboard/agent.go +++ b/internal/onboard/agent.go @@ -7,184 +7,17 @@ package onboard import ( "context" - "encoding/json" + "errors" "fmt" "io" "os" - "os/exec" - "path/filepath" - "runtime" "slices" "strings" - "errors" "github.com/DataDog/ddtest/internal/platform" "github.com/kballard/go-shellquote" ) -// AgentResult is the shared, versioned onboarding/testdrive protocol. Commands -// are argv, not shell programs. Failed checks never become agent decisions. -type PrerequisiteFact struct { - ForCommand string `json:"for_command"` - Directory string `json:"directory,omitempty"` - Argv []string `json:"argv"` - ExpectedFiles []string `json:"expected_files"` -} -type SetupFacts struct { - SchemaVersion int `json:"schema_version"` - Fingerprint string `json:"input_fingerprint"` - Prerequisites []PrerequisiteFact `json:"prerequisites"` - Configurations []ConfigurationFact `json:"configurations,omitempty"` -} - -type AgentResult struct { - RunID string `json:"run_id,omitempty"` - Plan *ExecutionPlan `json:"plan,omitempty"` - PlanSummary *PlanSummary `json:"plan_summary,omitempty"` - DetailsCommand []string `json:"details_command,omitempty"` - InputTemplate *SetupFacts `json:"input_template,omitempty"` - SchemaVersion int `json:"schema_version"` - Status string `json:"status"` - Phase string `json:"phase"` - Code string `json:"code,omitempty"` - Reason string `json:"reason,omitempty"` - Location string `json:"location,omitempty"` - Needs string `json:"needs,omitempty"` - Capabilities []string `json:"required_capabilities,omitempty"` - RequiredInput []string `json:"required_input,omitempty"` - NextAction string `json:"next_action,omitempty"` - NextCommand []string `json:"next_command,omitempty"` - // SetupCommands are the argv commands, run in order from the repository - // root, that prepare dependencies before resume_command; next_command is - // the first of them. - SetupCommands [][]string `json:"setup_commands,omitempty"` - ResumeCommand []string `json:"resume_command,omitempty"` - // AcceptResumeCommand is resume_command plus a decision only the user may - // make, such as accepting a substitute runtime. - AcceptResumeCommand []string `json:"accept_resume_command,omitempty"` - Instruction string `json:"agent_instruction,omitempty"` - Fingerprint string `json:"input_fingerprint,omitempty"` - Checks []WorkflowCheck `json:"checks,omitempty"` - SetupDiff string `json:"setup_diff,omitempty"` - UndoInstructions string `json:"undo_instructions,omitempty"` - Changes []SetupChange `json:"changes,omitempty"` - FinalResponse string `json:"final_response,omitempty"` - JSONReport string `json:"json_report,omitempty"` - HTMLReport string `json:"html_report,omitempty"` -} - -// PlanSummary keeps routine handoffs independent of CI matrix size. -type PlanSummary struct { - UnsupportedFeatures map[string]string `json:"unsupported_features,omitempty"` - KnownLimitations map[string]string `json:"known_limitations,omitempty"` - Features []string `json:"feature_scenarios,omitempty"` - Framework string `json:"framework"` - Tracer string `json:"tracer,omitempty"` - Site string `json:"site"` - Initialization string `json:"initialization"` - Configurations int `json:"configurations"` - Unresolved int `json:"unresolved"` -} - -// WorkflowCheck records an owned check and its command, without unbounded logs. -type WorkflowCheck struct { - Name string `json:"name"` - Status string `json:"status"` - Command []string `json:"command,omitempty"` - Reason string `json:"reason,omitempty"` -} - -func WriteAgentResult(output io.Writer, result AgentResult) error { - result.SchemaVersion = 2 - return json.NewEncoder(output).Encode(result) -} - -// RenderAgentResult gives hosts a structured response or renders its facts -// directly in the terminal. Rendering never asks a model to recompute a verdict. -func RenderAgentResult(output io.Writer, result AgentResult, format string) error { - if result.Plan != nil { - plan := result.Plan - result.PlanSummary = &PlanSummary{Framework: plan.Framework, Tracer: plan.Tracer, Site: plan.Site, Initialization: plan.Initialization, Configurations: len(plan.Scope.Commands), Unresolved: len(plan.Scope.Unresolved), Features: plan.Features, UnsupportedFeatures: plan.UnsupportedFeatures, KnownLimitations: plan.KnownLimitations} - result.DetailsCommand = DDTestCommand("testdrive", "--report", "--detail", "plan") - if format == "json" && len(result.DetailsCommand) > 0 { - result.DetailsCommand = append(result.DetailsCommand, "--format", "json") - } - if len(result.DetailsCommand) > 0 && result.RunID != "" && result.Status != "complete_locally" && result.Status != "selected_passed" { - result.DetailsCommand = append(result.DetailsCommand, "--resume", result.RunID) - } - result.Plan = nil - } - - if result.FinalResponse != "" && !strings.Contains(result.Instruction, FinalResponseInstruction) { - // First, so it is read before the rest of the instruction. - result.Instruction = strings.TrimSpace(FinalResponseInstruction + " " + result.Instruction) - } - if !strings.Contains(result.Instruction, ReadWholeInstruction) && !strings.Contains(result.Reason, ReadWholeInstruction) { - result.Instruction = strings.TrimSpace(result.Instruction + " " + ReadWholeInstruction) - } - if format == "verbose" { - return writeAgentText(output, result) - } - return WriteAgentResult(output, result) -} - -// writeAgentText renders a result for a person reading the terminal. Hosts -// select the agent or json format to receive the structured document. -func writeAgentText(output io.Writer, result AgentResult) error { - var text strings.Builder - status := result.Status - if result.Code != "" { - status += " (" + result.Code + ")" - } - _, _ = fmt.Fprintf(&text, "\nResult: %s, phase %s\n", status, result.Phase) - for _, line := range []struct{ label, value string }{{"Reason", result.Reason}, {"Location", result.Location}, {"Needs", result.Needs}} { - if line.value != "" { - _, _ = fmt.Fprintf(&text, "%s: %s\n", line.label, line.value) - } - } - for _, input := range result.RequiredInput { - _, _ = fmt.Fprintf(&text, "Required input: %s\n", input) - } - for _, command := range result.SetupCommands { - _, _ = fmt.Fprintf(&text, "Setup: %s\n", shellquote.Join(command...)) - } - for _, change := range result.Changes { - _, _ = fmt.Fprintf(&text, "Changed: %s\n", change.Path) - } - for _, check := range result.Checks { - line := check.Name + ": " + check.Status - if check.Reason != "" { - line += " — " + check.Reason - } - _, _ = fmt.Fprintf(&text, "Check %s\n", line) - } - if result.InputTemplate != nil { - data, err := json.MarshalIndent(result.InputTemplate, "", " ") - if err != nil { - return err - } - _, _ = fmt.Fprintf(&text, "Facts template (complete it, save it outside the checkout, then pass --facts ):\n%s\n", data) - } - for _, line := range []struct { - label string - command []string - }{{"Next", result.NextCommand}, {"Resume", result.ResumeCommand}, {"After user approval", result.AcceptResumeCommand}, {"Details", result.DetailsCommand}} { - if len(line.command) > 0 { - _, _ = fmt.Fprintf(&text, "%s: %s\n", line.label, shellquote.Join(line.command...)) - } - } - for _, report := range []struct{ label, path string }{{"JSON report", result.JSONReport}, {"HTML report", result.HTMLReport}} { - if report.path != "" { - _, _ = fmt.Fprintf(&text, "%s: %s\n", report.label, report.path) - } - } - if result.FinalResponse != "" { - _, _ = fmt.Fprintf(&text, "\nSummary:\n%s\n", result.FinalResponse) - } - _, err := io.WriteString(output, text.String()) - return err -} - type StartOptions struct { Format string Site string @@ -511,102 +344,9 @@ const ReadWholeInstruction = "Each result is one bounded JSON document: read it // paraphrases turning an incomplete verdict into a completion claim. const FinalResponseInstruction = "Begin your answer with final_response exactly, as its first block, and add context only after it. When it starts with NOT COMPLETE, do not call onboarding complete or successful, and do not record that result as expected or final in notes or memory." -// StandaloneVerdictInstruction states what a standalone validation's verdict -// certifies, after FinalResponseInstruction: trials called a pytest PASSED, -// and an incomplete report beside a passing local validation, done. -func StandaloneVerdictInstruction(framework string) string { - return framework + " validation is standalone: its passing verdict is PASSED, never COMPLETE LOCALLY, and only COMPLETE LOCALLY certifies onboarding as complete. Report a PASSED verdict as validated locally, in final_response's words; do not call onboarding done, complete or successful." -} - // humanOnboardingHandoff points at the preview, which writes nothing, before // the command that applies the edits and runs local validation. func humanOnboardingHandoff(framework string, command []string) string { preview := slices.DeleteFunc(slices.Clone(command), func(arg string) bool { return arg == "--yes" }) return fmt.Sprintf("Detected %s. Preview the recognized CI setup first; this writes nothing:\n %s\nThen apply it and validate it locally:\n %s\n", framework, shellquote.Join(preview...), shellquote.Join(command...)) } - -// A go-run executable disappears after the command exits. It cannot be used in -// a protocol handoff; require a persistent build unless one is already on PATH. -const TransientExecutableReason = "The temporary go run executable cannot be reused. Run make build in the ddtest source checkout, then invoke that persistent ddtest binary to continue." - -func persistentExecutable(path string) bool { - if filepath.Base(filepath.Dir(path)) != "exe" { - return true - } - for directory := filepath.Dir(path); filepath.Dir(directory) != directory; directory = filepath.Dir(directory) { - if strings.HasPrefix(filepath.Base(directory), "go-build") { - return false - } - } - return true -} - -func DDTestCommand(args ...string) []string { - executable, _ := ddtestExecutable() - if executable == "" { - return nil - } - command := append([]string{executable}, args...) - // Follow-up commands keep the selected CI provider. - if provider := ActiveCIProvider(); provider != "" && len(args) > 0 && (args[0] == "onboard" || args[0] == "testdrive") && !slices.Contains(args, "--ci-provider") { - command = append(command, "--ci-provider", provider) - } - // And the user's statements about CI, which reports are bound to. - if len(args) > 0 && (args[0] == "onboard" || args[0] == "testdrive") && !slices.Contains(args, "--ci-runtime") && !slices.Contains(args, "--ci-confirm") { - command = append(command, CIStatementArguments()...) - } - return command -} - -// SelfPathVariable names an executable that returned commands use in place of -// the running one, such as a wrapper that runs this ddtest. -const SelfPathVariable = "DDTEST_SELF_PATH" - -// selfPath is SelfPathVariable's value when it is an absolute path to an -// executable regular file; empty otherwise. -func selfPath() string { - path := os.Getenv(SelfPathVariable) - if !filepath.IsAbs(path) { - return "" - } - info, err := os.Stat(path) - if err != nil || !info.Mode().IsRegular() || (runtime.GOOS != "windows" && info.Mode().Perm()&0o111 == 0) { - return "" - } - return path -} - -// ddtestExecutable prefers SelfPathVariable, then the running executable. Only -// a temporary go run binary falls back to ddtest on PATH, which may be a -// different version. -func ddtestExecutable() (path string, fromPATH bool) { - if path := selfPath(); path != "" { - return path, false - } - executable, err := os.Executable() - if err == nil && persistentExecutable(executable) { - return executable, false - } - executable, err = exec.LookPath("ddtest") - if err != nil || !persistentExecutable(executable) { - return "", false - } - return executable, true -} - -// ExecutableFallbackNotice says when handoff commands use ddtest from PATH -// instead of the running executable; empty otherwise. -func ExecutableFallbackNotice() string { - if executable, fromPATH := ddtestExecutable(); fromPATH { - return "This executable is temporary (go run), so commands use ddtest from PATH (" + executable + "), which may be a different version." - } - return "" -} - -func OnboardingCommand(frameworks ...string) []string { - framework := "vitest" - if len(frameworks) > 0 { - framework = frameworks[0] - } - return DDTestCommand("onboard", "--framework", framework, "--apply", "--yes") -} diff --git a/internal/onboard/agent_result.go b/internal/onboard/agent_result.go new file mode 100644 index 000000000..3f7a5e29f --- /dev/null +++ b/internal/onboard/agent_result.go @@ -0,0 +1,172 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/kballard/go-shellquote" +) + +// AgentResult is the shared, versioned onboarding/testdrive protocol. Commands +// are argv, not shell programs. Failed checks never become agent decisions. +type AgentResult struct { + RunID string `json:"run_id,omitempty"` + Plan *ExecutionPlan `json:"plan,omitempty"` + PlanSummary *PlanSummary `json:"plan_summary,omitempty"` + DetailsCommand []string `json:"details_command,omitempty"` + InputTemplate *SetupFacts `json:"input_template,omitempty"` + SchemaVersion int `json:"schema_version"` + Status string `json:"status"` + Phase string `json:"phase"` + Code string `json:"code,omitempty"` + Reason string `json:"reason,omitempty"` + Location string `json:"location,omitempty"` + Needs string `json:"needs,omitempty"` + Capabilities []string `json:"required_capabilities,omitempty"` + RequiredInput []string `json:"required_input,omitempty"` + NextAction string `json:"next_action,omitempty"` + NextCommand []string `json:"next_command,omitempty"` + // SetupCommands are the argv commands, run in order from the repository + // root, that prepare dependencies before resume_command; next_command is + // the first of them. + SetupCommands [][]string `json:"setup_commands,omitempty"` + ResumeCommand []string `json:"resume_command,omitempty"` + // AcceptResumeCommand is resume_command plus a decision only the user may + // make, such as accepting a substitute runtime. + AcceptResumeCommand []string `json:"accept_resume_command,omitempty"` + Instruction string `json:"agent_instruction,omitempty"` + Fingerprint string `json:"input_fingerprint,omitempty"` + Checks []WorkflowCheck `json:"checks,omitempty"` + SetupDiff string `json:"setup_diff,omitempty"` + UndoInstructions string `json:"undo_instructions,omitempty"` + Changes []SetupChange `json:"changes,omitempty"` + FinalResponse string `json:"final_response,omitempty"` + JSONReport string `json:"json_report,omitempty"` + HTMLReport string `json:"html_report,omitempty"` +} + +// PlanSummary keeps routine handoffs independent of CI matrix size. +type PlanSummary struct { + UnsupportedFeatures map[string]string `json:"unsupported_features,omitempty"` + KnownLimitations map[string]string `json:"known_limitations,omitempty"` + Features []string `json:"feature_scenarios,omitempty"` + Framework string `json:"framework"` + Tracer string `json:"tracer,omitempty"` + Site string `json:"site"` + Initialization string `json:"initialization"` + Configurations int `json:"configurations"` + Unresolved int `json:"unresolved"` +} + +// WorkflowCheck records an owned check and its command, without unbounded logs. +type WorkflowCheck struct { + Name string `json:"name"` + Status string `json:"status"` + Command []string `json:"command,omitempty"` + Reason string `json:"reason,omitempty"` +} + +func WriteAgentResult(output io.Writer, result AgentResult) error { + result.SchemaVersion = 2 + return json.NewEncoder(output).Encode(result) +} + +// RenderAgentResult gives hosts a structured response or renders its facts +// directly in the terminal. Rendering never asks a model to recompute a verdict. +func RenderAgentResult(output io.Writer, result AgentResult, format string) error { + if result.Plan != nil { + plan := result.Plan + result.PlanSummary = &PlanSummary{Framework: plan.Framework, Tracer: plan.Tracer, Site: plan.Site, Initialization: plan.Initialization, Configurations: len(plan.Scope.Commands), Unresolved: len(plan.Scope.Unresolved), Features: plan.Features, UnsupportedFeatures: plan.UnsupportedFeatures, KnownLimitations: plan.KnownLimitations} + result.DetailsCommand = DDTestCommand("testdrive", "--report", "--detail", "plan") + if format == "json" && len(result.DetailsCommand) > 0 { + result.DetailsCommand = append(result.DetailsCommand, "--format", "json") + } + if len(result.DetailsCommand) > 0 && result.RunID != "" && result.Status != "complete_locally" && result.Status != "selected_passed" { + result.DetailsCommand = append(result.DetailsCommand, "--resume", result.RunID) + } + result.Plan = nil + } + + if result.FinalResponse != "" && !strings.Contains(result.Instruction, FinalResponseInstruction) { + // First, so it is read before the rest of the instruction. + result.Instruction = strings.TrimSpace(FinalResponseInstruction + " " + result.Instruction) + } + if !strings.Contains(result.Instruction, ReadWholeInstruction) && !strings.Contains(result.Reason, ReadWholeInstruction) { + result.Instruction = strings.TrimSpace(result.Instruction + " " + ReadWholeInstruction) + } + if format == "verbose" { + return writeAgentText(output, result) + } + return WriteAgentResult(output, result) +} + +// writeAgentText renders a result for a person reading the terminal. Hosts +// select the agent or json format to receive the structured document. +func writeAgentText(output io.Writer, result AgentResult) error { + var text strings.Builder + status := result.Status + if result.Code != "" { + status += " (" + result.Code + ")" + } + _, _ = fmt.Fprintf(&text, "\nResult: %s, phase %s\n", status, result.Phase) + for _, line := range []struct{ label, value string }{{"Reason", result.Reason}, {"Location", result.Location}, {"Needs", result.Needs}} { + if line.value != "" { + _, _ = fmt.Fprintf(&text, "%s: %s\n", line.label, line.value) + } + } + for _, input := range result.RequiredInput { + _, _ = fmt.Fprintf(&text, "Required input: %s\n", input) + } + for _, command := range result.SetupCommands { + _, _ = fmt.Fprintf(&text, "Setup: %s\n", shellquote.Join(command...)) + } + for _, change := range result.Changes { + _, _ = fmt.Fprintf(&text, "Changed: %s\n", change.Path) + } + for _, check := range result.Checks { + line := check.Name + ": " + check.Status + if check.Reason != "" { + line += " — " + check.Reason + } + _, _ = fmt.Fprintf(&text, "Check %s\n", line) + } + if result.InputTemplate != nil { + data, err := json.MarshalIndent(result.InputTemplate, "", " ") + if err != nil { + return err + } + _, _ = fmt.Fprintf(&text, "Facts template (complete it, save it outside the checkout, then pass --facts ):\n%s\n", data) + } + for _, line := range []struct { + label string + command []string + }{{"Next", result.NextCommand}, {"Resume", result.ResumeCommand}, {"After user approval", result.AcceptResumeCommand}, {"Details", result.DetailsCommand}} { + if len(line.command) > 0 { + _, _ = fmt.Fprintf(&text, "%s: %s\n", line.label, shellquote.Join(line.command...)) + } + } + for _, report := range []struct{ label, path string }{{"JSON report", result.JSONReport}, {"HTML report", result.HTMLReport}} { + if report.path != "" { + _, _ = fmt.Fprintf(&text, "%s: %s\n", report.label, report.path) + } + } + if result.FinalResponse != "" { + _, _ = fmt.Fprintf(&text, "\nSummary:\n%s\n", result.FinalResponse) + } + _, err := io.WriteString(output, text.String()) + return err +} + +// StandaloneVerdictInstruction states what a standalone validation's verdict +// certifies, after FinalResponseInstruction: trials called a pytest PASSED, +// and an incomplete report beside a passing local validation, done. +func StandaloneVerdictInstruction(framework string) string { + return framework + " validation is standalone: its passing verdict is PASSED, never COMPLETE LOCALLY, and only COMPLETE LOCALLY certifies onboarding as complete. Report a PASSED verdict as validated locally, in final_response's words; do not call onboarding done, complete or successful." +} diff --git a/internal/onboard/commands.go b/internal/onboard/commands.go new file mode 100644 index 000000000..ba67cd4c0 --- /dev/null +++ b/internal/onboard/commands.go @@ -0,0 +1,101 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" +) + +// A go-run executable disappears after the command exits. It cannot be used in +// a protocol handoff; require a persistent build unless one is already on PATH. +const TransientExecutableReason = "The temporary go run executable cannot be reused. Run make build in the ddtest source checkout, then invoke that persistent ddtest binary to continue." + +func persistentExecutable(path string) bool { + if filepath.Base(filepath.Dir(path)) != "exe" { + return true + } + for directory := filepath.Dir(path); filepath.Dir(directory) != directory; directory = filepath.Dir(directory) { + if strings.HasPrefix(filepath.Base(directory), "go-build") { + return false + } + } + return true +} + +func DDTestCommand(args ...string) []string { + executable, _ := ddtestExecutable() + if executable == "" { + return nil + } + command := append([]string{executable}, args...) + // Follow-up commands keep the selected CI provider. + if provider := ActiveCIProvider(); provider != "" && len(args) > 0 && (args[0] == "onboard" || args[0] == "testdrive") && !slices.Contains(args, "--ci-provider") { + command = append(command, "--ci-provider", provider) + } + // And the user's statements about CI, which reports are bound to. + if len(args) > 0 && (args[0] == "onboard" || args[0] == "testdrive") && !slices.Contains(args, "--ci-runtime") && !slices.Contains(args, "--ci-confirm") { + command = append(command, CIStatementArguments()...) + } + return command +} + +// SelfPathVariable names an executable that returned commands use in place of +// the running one, such as a wrapper that runs this ddtest. +const SelfPathVariable = "DDTEST_SELF_PATH" + +// selfPath is SelfPathVariable's value when it is an absolute path to an +// executable regular file; empty otherwise. +func selfPath() string { + path := os.Getenv(SelfPathVariable) + if !filepath.IsAbs(path) { + return "" + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || (runtime.GOOS != "windows" && info.Mode().Perm()&0o111 == 0) { + return "" + } + return path +} + +// ddtestExecutable prefers SelfPathVariable, then the running executable. Only +// a temporary go run binary falls back to ddtest on PATH, which may be a +// different version. +func ddtestExecutable() (path string, fromPATH bool) { + if path := selfPath(); path != "" { + return path, false + } + executable, err := os.Executable() + if err == nil && persistentExecutable(executable) { + return executable, false + } + executable, err = exec.LookPath("ddtest") + if err != nil || !persistentExecutable(executable) { + return "", false + } + return executable, true +} + +// ExecutableFallbackNotice says when handoff commands use ddtest from PATH +// instead of the running executable; empty otherwise. +func ExecutableFallbackNotice() string { + if executable, fromPATH := ddtestExecutable(); fromPATH { + return "This executable is temporary (go run), so commands use ddtest from PATH (" + executable + "), which may be a different version." + } + return "" +} + +func OnboardingCommand(frameworks ...string) []string { + framework := "vitest" + if len(frameworks) > 0 { + framework = frameworks[0] + } + return DDTestCommand("onboard", "--framework", framework, "--apply", "--yes") +} diff --git a/internal/onboard/execution_plan.go b/internal/onboard/execution_plan.go index 25ba666a8..ad5d0ea59 100644 --- a/internal/onboard/execution_plan.go +++ b/internal/onboard/execution_plan.go @@ -59,6 +59,21 @@ func AssignConfigurationIDs(scope *ValidationScope) { } } +// Setup facts describe local prerequisites bound to an input fingerprint. +// They add setup to discovered commands without overriding validation. +type PrerequisiteFact struct { + ForCommand string `json:"for_command"` + Directory string `json:"directory,omitempty"` + Argv []string `json:"argv"` + ExpectedFiles []string `json:"expected_files"` +} +type SetupFacts struct { + SchemaVersion int `json:"schema_version"` + Fingerprint string `json:"input_fingerprint"` + Prerequisites []PrerequisiteFact `json:"prerequisites"` + Configurations []ConfigurationFact `json:"configurations,omitempty"` +} + // ServiceCommand describes a repository-owned application, not the test runner. // PortEnvironment is an existing shared app/test override. A free port is chosen // only when that explicit override is supplied; arbitrary config rewriting is not. diff --git a/internal/testdrive/standalone_workflow.go b/internal/testdrive/standalone_workflow.go index 007420dc5..1aeaf835e 100644 --- a/internal/testdrive/standalone_workflow.go +++ b/internal/testdrive/standalone_workflow.go @@ -23,7 +23,9 @@ import ( // results, then runs the standalone validation and returns its verdict with // the report's final_response. It keeps no checkpoint, so it takes neither // --resume nor --facts. -func (t *Testdrive) standaloneWorkflow(ctx context.Context, output io.Writer, options WorkflowOptions, result *onboard.AgentResult, recoveryCommand []string, fail func(status, code, reason, needs string) error, finalResponse *string) error { +func (t *Testdrive) standaloneWorkflow(ctx context.Context, attempt *workflowAttempt, fail func(status, code, reason, needs string) error) error { + output, options := attempt.output, attempt.options + result, recoveryCommand := attempt.result, attempt.recoveryCommand name := t.framework.Name() if options.Resume != "" || options.FactsFile != "" { result.NextCommand = nil @@ -48,15 +50,8 @@ func (t *Testdrive) standaloneWorkflow(ctx context.Context, output io.Writer, op return onboard.RenderAgentResult(output, *result, options.Format) } // Only the user can tell these; nothing runs before they answer. - if required := onboard.RequiredCIRuntimes(ctx, t.repositoryRoot, t.language, name); required != nil { - asked := onboard.CIRuntimeRequiredResult(required) - result.Location, result.RequiredInput, result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction - return fail("needs_input", asked.Code, asked.Reason, asked.Instruction) - } - if required := onboard.RequiredCIConfirmations(t.repositoryRoot, t.language, name); required != nil { - asked := onboard.CIConfirmationRequiredResult(required) - result.Location, result.RequiredInput, result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction - return fail("needs_input", asked.Code, asked.Reason, asked.Instruction) + if err := attempt.checkCIStatements(ctx, t); err != nil { + return err } // A test command that drops what the setup sets runs untraced in CI: // the exact edit comes first, then this same command. @@ -67,16 +62,9 @@ func (t *Testdrive) standaloneWorkflow(ctx context.Context, output io.Writer, op } if !options.ExplicitVersion && t.projectTracer == "" { // The handoff cannot name the release the CI setup pins; validate it. - pin := "" - for _, job := range t.lang().ciRuntimes(ctx, t.repositoryRoot, name).Jobs { - if candidate := t.lang().ciPinnedTracer(job); candidate != "" { - if pin != "" && pin != candidate { - result.Location = job.Workflow + " / " + job.Job - result.RequiredInput = []string{"one intended CI tracer release, or separate explicitly scoped runs"} - return fail("needs_input", "multiple_ci_tracers", "CI selects different tracer releases.", "Preserve every configuration and align the intended tracer selection explicitly.") - } - pin = candidate - } + pin, err := attempt.ciPinnedTracer(ctx, t) + if err != nil { + return err } if pin != "" && pin != t.tracerVersion { prepared, err := Prepare(PrepareOptions{TracerVersion: pin, CheckOnly: t.checkOnly, All: t.allConfigurations, Timeout: t.commandTimeout, AllowReviewedBuilds: t.allowReviewedBuilds, AcceptSubstituteRuntime: t.acceptSubstituteRuntime}) @@ -122,7 +110,7 @@ func (t *Testdrive) standaloneWorkflow(ctx context.Context, output io.Writer, op result.FinalResponse = validation.Summary.FinalResponse return onboard.RenderAgentResult(output, *result, options.Format) } - *finalResponse = validation.Summary.FinalResponse + attempt.finalResponse = validation.Summary.FinalResponse result.ResumeCommand = append([]string{}, recoveryCommand...) result.RequiredInput = nil reportOnly, actionable := splitTracerOnlyBlockers(validation) diff --git a/internal/testdrive/workflow.go b/internal/testdrive/workflow.go index c8a7ede96..d8bf10f9a 100644 --- a/internal/testdrive/workflow.go +++ b/internal/testdrive/workflow.go @@ -7,12 +7,9 @@ package testdrive import ( "context" - "crypto/sha256" - "encoding/json" "errors" "fmt" "io" - "net/url" "os" "slices" "strings" @@ -44,129 +41,18 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work result.Status, result.Code, result.Reason, result.NextAction = "needs_input", "transient_executable", onboard.TransientExecutableReason, "supply_input" return errors.Join(fmt.Errorf("%s", result.Reason), onboard.RenderAgentResult(output, result, options.Format)) } - recoveryCommand := append([]string{}, result.NextCommand...) - var checkpoint *workflowCheckpoint - var factsHash string - ownedLock, executed := false, false - // acceptFlag, when set, is appended to the final resume command after the - // checkpoint rewrites --resume, so the offered command never goes stale. - acceptFlag := "" - turnFinalResponse := "" - // attemptStarted is when this attempt's validation began, so a failure - // reads only the report it wrote. - var attemptStarted time.Time + attempt := &workflowAttempt{ + result: &result, output: output, options: options, framework: name, + recoveryCommand: append([]string{}, result.NextCommand...), + } + // Preparation may replace t; failures must use the currently prepared drive. fail := func(status, code, reason, needs string) error { - result.Status, result.Code, result.Reason, result.Needs = status, code, reportText(reason), needs - if options.Format == "verbose" && result.UndoInstructions != "" { - _, _ = io.WriteString(output, result.UndoInstructions) - } - if checkpoint != nil { - if checkpoint.FactsSHA256 == "" || (code != "facts_invalid" && code != "facts_refresh_required") { - checkpoint.Plan = result.Plan - } - var varying []string - if !attemptStarted.IsZero() { - if report, err := readCurrentOnboardingResult(t.repositoryRoot, attemptStarted); err == nil { - varying = varyingTests(report) - } - } - if err := checkpoint.recordFailure(result.Fingerprint, factsHash, code, result.Reason, varying...); err != nil { - result.Reason += "; checkpoint: " + err.Error() - } - result.RunID = checkpoint.RunID - if result.ResumeCommand == nil { - result.ResumeCommand = append(append([]string{}, recoveryCommand...), "--resume", checkpoint.RunID) - } else { - for i := 0; i+1 < len(result.ResumeCommand); i++ { - if result.ResumeCommand[i] == "--resume" { - result.ResumeCommand[i+1] = checkpoint.RunID - } - } - } - } - if acceptFlag != "" && len(result.ResumeCommand) > 0 && !slices.Contains(result.ResumeCommand, acceptFlag) { - result.AcceptResumeCommand = append(append([]string{}, result.ResumeCommand...), acceptFlag) - } - switch status { - case "blocked_environment": - switch code { - case "workflow_busy": - result.NextAction = "wait" - case fileDescriptorsExhaustedCode: - // A transient host limit, not a missing capability: retry as is. - result.NextAction = "run_command" - result.NextCommand = append([]string{}, recoveryCommand...) - if len(result.ResumeCommand) > 0 { - result.NextCommand = append([]string{}, result.ResumeCommand...) - } - default: - result.NextAction = "request_capability" - } - case "needs_input": - switch code { - case "scope_unresolved": - result.NextAction = "run_command" - result.NextCommand = onboard.DDTestCommand("onboard", "--framework", name, "--instructions") - result.ResumeCommand, result.RequiredInput = nil, nil - result.Needs = scopeUnresolvedNeeds(result.Needs, options.Site) - case "setup_dependencies_missing", "project_dependencies_missing": - result.NextAction = "run_command" - case "ci_edit_required", "report_exclusions_required": - result.NextAction = "apply_source_edit" - case "ci_runtime_required", "ci_setup_confirmation_required": - // Only the user's answer, added to the same command, continues. - result.NextAction = "ask_user" - result.ResumeCommand = nil - default: - result.NextAction = "supply_input" - } - case "unsupported": - if code == "ci_edit_unsupported" && len(result.NextCommand) > 0 { - // The setup to apply comes from next_command (the manual - // instructions); resume_command validates it. - result.NextAction = "run_command" - } else if code == "ci_edit_unsupported" { - result.NextAction = "apply_source_edit" - } else { - result.NextAction = "report_result" - } - default: - result.NextAction = "report_result" - } - if ownedLock && !executed && !t.checkOnly && !workflowInputFailure(code) { - invalidatePriorWorkflow(t.repositoryRoot, name, result) - } - switch result.NextAction { - case "report_result", "supply_input", "apply_source_edit", "wait", "ask_user": - result.NextCommand = nil - case "request_capability": - // Without a checkpoint there is nothing to resume; rerun from the start. - if len(result.ResumeCommand) > 0 { - result.NextCommand = append([]string{}, result.ResumeCommand...) - } else { - result.NextCommand = append([]string{}, recoveryCommand...) - } - } - if checkpoint != nil && checkpoint.repeated && !transientWorkflowFailure(code) { - // A deterministic failure with unchanged inputs repeats on every rerun. - result.NextAction = "report_result" - result.NextCommand, result.ResumeCommand = nil, nil - result.Needs += " This attempt failed exactly like the previous one with unchanged inputs, so rerunning cannot progress; stop and report this blocker." - } else if checkpoint != nil && checkpoint.Attempts >= maxWorkflowAttempts { - result.NextAction = "report_result" - result.NextCommand, result.ResumeCommand = nil, nil - result.Needs += " Three attempts with unchanged inputs made no progress; stop and report this blocker." - } - // A setup repair continues the workflow; a report or a user decision ends - // the turn and must carry the text to relay. - if turnFinalResponse != "" && (result.NextAction == "report_result" || code == "substitute_runtime_unaccepted") { - result.FinalResponse = turnFinalResponse - } - return errors.Join(fmt.Errorf("%s: %s", code, result.Reason), onboard.RenderAgentResult(output, result, options.Format)) + return attempt.fail(t, status, code, reason, needs) } + if supportsValidationFramework(name) && StructuredValidation(t.language) && !t.lang().coordinated() && !options.Apply { // Standalone validation has no automatic CI edits; --apply stays unsupported. - return t.standaloneWorkflow(ctx, output, options, &result, recoveryCommand, fail, &turnFinalResponse) + return t.standaloneWorkflow(ctx, attempt, fail) } if !supportsValidationFramework(name) || !t.lang().coordinated() { result.NextCommand = nil @@ -186,7 +72,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work result.Capabilities = []string{"writable private temporary coordination storage"} return fail("blocked_environment", "coordination_unavailable", err.Error(), "Restore access/free space in the named temporary storage, then rerun next_command.") } - ownedLock = true + attempt.ownedLock = true ctx = context.WithValue(ctx, workflowLockKey{}, t.repositoryRoot) defer func() { _ = lock.Close() }() if err := cleanupExpiredCheckpoints(); err != nil { @@ -201,14 +87,14 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if err := recoverProbes(t.repositoryRoot); err != nil { return fail("needs_input", "probe_recovery_failed", err.Error(), "Review only the named owned probe; modified files are preserved. Then retry the same command.") } - checkpoint, err = openCheckpoint(options.Resume, t.repositoryRoot, name, settings.GetCommand(), options.Apply, t.allConfigurations) + attempt.checkpoint, err = openCheckpoint(options.Resume, t.repositoryRoot, name, settings.GetCommand(), options.Apply, t.allConfigurations) if err != nil { if errors.Is(err, errCheckpointNotFound) { return fail("needs_input", "resume_not_found", err.Error(), "Start a fresh workflow without --resume; this run already finished or its checkpoint expired.") } return fail("needs_input", "resume_invalid", err.Error(), "Use the supplied resume command with its original repository/framework/scope, or start a fresh workflow.") } - result.RunID = checkpoint.RunID + result.RunID = attempt.checkpoint.RunID progress := func(phase string) { result.Phase = phase if options.Format == "verbose" { @@ -275,62 +161,11 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work result.Plan = plan // A CI job's runtime that neither its configuration nor a statement // names is the user's answer to give; nothing is set up or run before it. - if required := onboard.RequiredCIRuntimes(ctx, t.repositoryRoot, t.language, name); required != nil { - asked := onboard.CIRuntimeRequiredResult(required) - result.Location, result.RequiredInput, result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction - return fail("needs_input", asked.Code, asked.Reason, asked.Instruction) - } - if required := onboard.RequiredCIConfirmations(t.repositoryRoot, t.language, name); required != nil { - asked := onboard.CIConfirmationRequiredResult(required) - result.Location, result.RequiredInput, result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction - return fail("needs_input", asked.Code, asked.Reason, asked.Instruction) - } - if options.FactsFile != "" { - factsFingerprint := fingerprint - // Built before facts touch the scope: rejected facts are rewritten from it. - template := scopeFactsTemplate(fingerprint, scope) - data, readErr := readSetupFacts(t.repositoryRoot, options.FactsFile) - factsHash = fmt.Sprintf("%x", sha256.Sum256(data)) - if readErr == nil && options.Resume == "" { - checkpoint.restoreFactsBinding(fingerprint, factsHash) - } - if readErr == nil && checkpoint.Fingerprint == fingerprint && checkpoint.FactsSHA256 == factsHash && checkpoint.Plan != nil && checkpoint.Engine == engineIdentity() { - // These exact facts already passed validation for the pre-edit inputs. - // Reuse only their additions, bound to the unchanged owned edit result, - // whose plan may still hold the pre-edit commands. - err = reattachSetupFacts(&scope, checkpoint.Plan.Scope, true) - factsFingerprint = checkpoint.FactsFingerprint - } else { - if readErr != nil { - err = readErr - } else { - _, err = applySetupFactsData(t.repositoryRoot, data, fingerprint, &scope) - } - } - - if err != nil && checkpoint.FactsDependencyHandoff && readErr == nil && checkpoint.FactsSHA256 == factsHash && checkpoint.Engine == engineIdentity() { - var refresh onboard.SetupFacts - if json.Unmarshal(data, &refresh) == nil { - refresh.Fingerprint = fingerprint - result.InputTemplate = &refresh - result.Location = options.FactsFile - result.RequiredInput = []string{"Review the unchanged setup entries against the post-install checkout, then replace the facts file with input_template and run resume_command."} - return fail("needs_input", "facts_refresh_required", "The declared dependency-install handoff changed repository inputs; the old facts cannot certify the new snapshot.", "Explicitly refresh facts for the returned current fingerprint; do not retry unchanged facts or bypass their binding.") - } - } - if err != nil { - result.Location = options.FactsFile - result.InputTemplate = template - result.RequiredInput = []string{"schema_version=2, current input_fingerprint, configurations with configuration_id and build/service setup"} - return fail("needs_input", "facts_invalid", err.Error(), "Rewrite the facts file outside the checkout from input_template for the current input_fingerprint, supplying only the missing prerequisite facts, then run resume_command. No discovery or validation assertion can be overridden.") - } - if checkpoint.FactsSHA256 != factsHash || checkpoint.Fingerprint != fingerprint || checkpoint.Engine != engineIdentity() { - checkpoint.Tracer = "" // Replacement setup invalidates cached preparation. - } - checkpoint.FactsSHA256 = factsHash - checkpoint.Fingerprint = fingerprint - checkpoint.Engine = engineIdentity() - checkpoint.FactsFingerprint = factsFingerprint + if err := attempt.checkCIStatements(ctx, t); err != nil { + return err + } + if err := attempt.acceptSetupFacts(t, fingerprint, &scope); err != nil { + return err } plan.Scope = scope if options.Apply || t.allConfigurations { @@ -379,17 +214,9 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work t.workflowScope = &scope } if !options.Apply && !options.ExplicitVersion && t.projectTracer == "" { - ci := t.lang().ciRuntimes(ctx, t.repositoryRoot, name) - pin := "" - for _, job := range ci.Jobs { - if candidate := t.lang().ciPinnedTracer(job); candidate != "" { - if pin != "" && pin != candidate { - result.Location = job.Workflow + " / " + job.Job - result.RequiredInput = []string{"one intended CI tracer release, or separate explicitly scoped runs"} - return fail("needs_input", "multiple_ci_tracers", "CI selects different tracer releases.", "Preserve every configuration and align the intended tracer selection explicitly.") - } - pin = candidate - } + pin, err := attempt.ciPinnedTracer(ctx, t) + if err != nil { + return err } if pin != "" { prepared, err := Prepare(PrepareOptions{TracerVersion: pin, All: t.allConfigurations, Timeout: t.commandTimeout, AllowReviewedBuilds: t.allowReviewedBuilds, AcceptSubstituteRuntime: t.acceptSubstituteRuntime}) @@ -402,54 +229,21 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if options.Apply { progress("preflight") - selected := checkpoint.Tracer + selected := attempt.checkpoint.Tracer var problem error - reused := checkpoint.reusable(fingerprint, t.workflowNodeVersion(), factsHash, t.tracerVersion, options.ExplicitVersion) + reused := attempt.checkpoint.reusable(fingerprint, t.workflowNodeVersion(), attempt.factsHash, t.tracerVersion, options.ExplicitVersion) if !reused { selectTracer := t.selectWorkflowTracer if t.workflowSelect != nil { selectTracer = t.workflowSelect } selected, problem = selectTracer(ctx, scope, options.ExplicitVersion) - } else if checkpoint.Plan != nil { - plan.CIRuntime = checkpoint.Plan.CIRuntime - plan.TracerNode = checkpoint.Plan.TracerNode + } else if attempt.checkpoint.Plan != nil { + plan.CIRuntime = attempt.checkpoint.Plan.CIRuntime + plan.TracerNode = attempt.checkpoint.Plan.TracerNode } if problem != nil { - var metadata *platform.MetadataError - if errors.As(problem, &metadata) { - result.Capabilities = []string{workflowMetadataCapability(metadata)} - return fail("blocked_environment", metadata.Code, problem.Error(), "Grant/fix only the named capability and rerun next_command. No tracer version was guessed and no source edits were made.") - } - var transport *url.Error - if errors.As(problem, &transport) { - address, _ := url.Parse(transport.URL) - capability := "CI metadata network access" - if address != nil { - capability = "network access to " + address.Hostname() - } - result.Capabilities = []string{capability} - return fail("blocked_environment", "metadata_network", problem.Error(), "Restore only the named metadata network capability, then run resume_command. Do not change runtime, tracer or test scope to work around a network failure.") - } - var majors *ciTracerMajors - if errors.As(problem, &majors) { - // Resume with the release ddtest recommends rather than the default. - if majors.Recommended != "" && !options.ExplicitVersion && checkpoint != nil { - result.ResumeCommand = append(append([]string{}, recoveryCommand...), "--tracer-version", majors.Recommended, "--resume", checkpoint.RunID) - } - if len(majors.Exclusions) > 0 { - result.RequiredInput = []string{"the user's choice: one --tracer-version every instrumented CI row supports, or approval to opt these rows out: " + strings.Join(majors.Exclusions, "; ")} - return fail("needs_input", "preflight_unresolved", problem.Error(), "Ask the user before excluding any CI row. If they approve, make exactly the listed DD_CIVISIBILITY_ENABLED edits and run resume_command; otherwise rerun with one --tracer-version every instrumented row supports. Never weaken feature checks.") - } - result.RequiredInput = []string{"one --tracer-version every instrumented CI row supports"} - needs := "No opt-out edit applies to these CI rows; do not exclude or rewrite them. Report them as unvalidated blockers unless one --tracer-version supports every instrumented row." - if majors.Recommended != "" { - needs = "Run resume_command, which selects dd-trace@" + majors.Recommended + ". Rows it cannot run stay unvalidated blockers to report; do not exclude or rewrite CI rows." - } - return fail("needs_input", "preflight_unresolved", problem.Error(), needs) - } - result.RequiredInput = []string{"the named runtime/configuration incompatibility"} - return fail("needs_input", "preflight_unresolved", problem.Error(), "Resolve this exact mismatch without weakening feature checks. Do not exclude or rewrite CI rows to get past it; if it can't be resolved, report it as a blocker.") + return attempt.failTracerSelection(t, problem) } if len(t.workflowReportOnly) > 0 { reason := "Not set up or validated locally; reported as blockers: " + strings.Join(t.workflowReportOnly, "; ") @@ -473,10 +267,10 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work } result.Location = "CI initialization" result.RequiredInput = []string{"the " + onboard.CIProviderDisplayName() + " setup next_command prints, exactly as shown, with " + setting} - result.ResumeCommand = t.workflowValidationResume(recoveryCommand[0], name, selected, options) - checkpoint.Apply = false - checkpoint.All = true - result.ResumeCommand = append(result.ResumeCommand, "--resume", checkpoint.RunID) + result.ResumeCommand = t.workflowValidationResume(attempt.recoveryCommand[0], name, selected, options) + attempt.checkpoint.Apply = false + attempt.checkpoint.All = true + result.ResumeCommand = append(result.ResumeCommand, "--resume", attempt.checkpoint.RunID) result.NextCommand = onboard.DDTestCommand("onboard", "--framework", name, "--instructions") return fail("unsupported", "ci_edit_unsupported", "onboard --apply does not write the "+displayName(name)+" CI setup on "+onboard.CIProviderDisplayName()+". next_command prints it: apply that setup exactly as shown, with "+setting+": what ddtest selected and resume_command validates.", "Run next_command, apply only the CI setup it prints, exactly as shown and with "+setting+", then run resume_command. Do not write a loader, hook or plugin integration of your own, and do not run testdrive or the tests first.") } @@ -493,7 +287,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work // The manifest and lockfile changed only through the install ddtest // prescribed: extend them without asking for --allow-dirty. var dirty *onboard.DirtyWorkflowGap - if errors.As(err, &dirty) && checkpoint != nil && onboard.OnlyPrescribedDependencyChanges(t.repositoryRoot, dirty.Files, checkpoint.DependencyPackages) { + if errors.As(err, &dirty) && attempt.checkpoint != nil && onboard.OnlyPrescribedDependencyChanges(t.repositoryRoot, dirty.Files, attempt.checkpoint.DependencyPackages) { result.Changes, err = sourcePlan.Apply(true) } } @@ -506,59 +300,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work } } if err != nil { - var dependencies *onboard.SetupDependencyGap - if errors.As(err, &dependencies) { - checkpoint.FactsDependencyHandoff = true - checkpoint.DependencyPackages = dependencies.Packages - checkpoint.Engine = engineIdentity() - checkpoint.FactsSHA256 = factsHash - checkpoint.Fingerprint = fingerprint - result.Location = "package.json / devDependencies" - result.NextCommand = dependencies.Command - result.ResumeCommand = append(append([]string{}, recoveryCommand...), "--resume", checkpoint.RunID) - result.RequiredInput = []string{"Run next_command once, as given and without asking, to update the manifest and lockfile, then resume_command. Preserve the selected framework and CI test commands."} - result.Instruction = onboard.PrescribedEditInstruction - return fail("needs_input", "setup_dependencies_missing", err.Error(), "The recognized coverage recipe needs only these dependencies; installing them is a ddtest-prescribed edit, listed with its revert in the final report. No CI edits have been made.") - } - var dirty *onboard.DirtyWorkflowGap - if errors.As(err, &dirty) { - result.Location = strings.Join(dirty.Files, ", ") - result.RequiredInput = []string{"Review setup_diff against the existing target-file edits. With approval to preserve and extend them, run resume_command with --allow-dirty; do not switch to testdrive or change scope."} - result.ResumeCommand = append(append([]string{}, recoveryCommand...), "--allow-dirty", "--resume", checkpoint.RunID) - return fail("needs_input", "dirty_workflow", err.Error(), "Commit or stash the listed target files, then rerun; add --allow-dirty only when the user approves extending those uncommitted edits.") - } - var runtimeGap *onboard.NodeRuntimeGap - if errors.As(err, &runtimeGap) { - result.Location = runtimeGap.Location - result.RequiredInput = []string{"A CI Node version that satisfies " + runtimeGap.Requirement + ", or an older tracer selected with --tracer-version"} - return fail("needs_input", "ci_runtime_incompatible", err.Error(), "Upgrade the CI Node version, or select an older tracer with --tracer-version, then rerun. No CI hook needs to be written by hand.") - } - var editGap *onboard.EditThenApplyGap - if errors.As(err, &editGap) { - // One source edit, then onboard --apply writes the setup itself: - // validating before that would check CI without the setup. - result.Location = editGap.Location - result.RequiredInput = []string{"Only the source edit reason names, at " + editGap.Location + " (and a dependency it names, through its install command); then run resume_command, which adds the Datadog CI setup and validates."} - result.ResumeCommand = append([]string{}, recoveryCommand...) - if !slices.Contains(result.ResumeCommand, "--allow-dirty") { - // It extends the uncommitted edit just made. - result.ResumeCommand = append(result.ResumeCommand, "--allow-dirty") - } - result.NextCommand = nil - return fail("needs_input", "ci_edit_required", err.Error(), "Make only the edit reason names, then run resume_command: onboard --apply adds the Datadog CI setup and validates. Do not run testdrive first."+manualSiteSentence(options.Site)) - } - var gap *onboard.SetupGap - if errors.As(err, &gap) { - result.Location = gap.Location - result.RequiredInput = []string{"CI tracer initialization at " + gap.Location} - result.ResumeCommand = t.workflowValidationResume(recoveryCommand[0], name, selected, options) - checkpoint.Apply = false - checkpoint.All = true - result.ResumeCommand = append(result.ResumeCommand, "--resume", checkpoint.RunID) - result.NextCommand = nil - return fail("unsupported", "ci_edit_unsupported", err.Error(), "Integrate only this CI hook, then run resume_command. Do not repeatedly retry the automatic editor or drop configurations."+manualSiteSentence(options.Site)) - } - return fail("blocked_environment", "ci_edit_unavailable", err.Error(), "Restore filesystem access. Completed source edits are listed in changes; do not overwrite customer edits.") + return attempt.failSetupEdits(t, err, selected, fingerprint) } // Bind the accepted plan and facts to the snapshot this owned edit produced // at once, so a later failure resumes with them instead of a dead end. @@ -567,9 +309,9 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work return fail("blocked_environment", "input_snapshot_unavailable", err.Error(), "Restore source input access. Completed source edits are listed in changes.") } result.Fingerprint = applied - checkpoint.Engine, checkpoint.Fingerprint, checkpoint.FactsSHA256, checkpoint.Plan = engineIdentity(), applied, factsHash, plan - checkpoint.Tracer, checkpoint.Node = selected, t.workflowNodeVersion() - if err := errors.Join(checkpoint.save(), checkpoint.saveFactsBinding()); err != nil { + attempt.checkpoint.Engine, attempt.checkpoint.Fingerprint, attempt.checkpoint.FactsSHA256, attempt.checkpoint.Plan = engineIdentity(), applied, attempt.factsHash, plan + attempt.checkpoint.Tracer, attempt.checkpoint.Node = selected, t.workflowNodeVersion() + if err := errors.Join(attempt.checkpoint.save(), attempt.checkpoint.saveFactsBinding()); err != nil { return fail("blocked_environment", "checkpoint_unavailable", err.Error(), "Restore private temporary storage. Completed source edits are listed in changes.") } // Refresh discovery after YAML insertion. Facts remain the exact additions @@ -599,18 +341,16 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work t = prepared.withWorkflowState(t) // A setup the edit just added may depend on facts only the CI server // shows; the user states them before anything runs. - if required := onboard.RequiredCIConfirmations(t.repositoryRoot, t.language, name); required != nil { - asked := onboard.CIConfirmationRequiredResult(required) - result.Location, result.RequiredInput, result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction - return fail("needs_input", asked.Code, asked.Reason, asked.Instruction) + if err := attempt.checkCIConfirmations(t); err != nil { + return err } } t.setPlanTracer(plan, t.tracerVersion) - checkpoint.Engine = engineIdentity() - checkpoint.Fingerprint, checkpoint.Node, checkpoint.FactsSHA256, checkpoint.Tracer, checkpoint.Plan = plan.Fingerprint, t.workflowNodeVersion(), factsHash, plan.Tracer, plan + attempt.checkpoint.Engine = engineIdentity() + attempt.checkpoint.Fingerprint, attempt.checkpoint.Node, attempt.checkpoint.FactsSHA256, attempt.checkpoint.Tracer, attempt.checkpoint.Plan = plan.Fingerprint, t.workflowNodeVersion(), attempt.factsHash, plan.Tracer, plan // Cache only successful selection, not failed preflight or a test verdict. if !options.Apply { - checkpoint.Tracer = "" + attempt.checkpoint.Tracer = "" } // Inventory dependencies before anything runs; the checks share one owned // copy for this invocation. @@ -623,7 +363,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if len(setup) > 0 { result.SetupCommands = setup result.NextCommand = setup[0] - result.ResumeCommand = append(append([]string{}, recoveryCommand...), "--resume", checkpoint.RunID) + result.ResumeCommand = append(append([]string{}, attempt.recoveryCommand...), "--resume", attempt.checkpoint.RunID) result.RequiredInput = []string{"Run every setup_commands entry once, in order, from the repository root, then run resume_command."} result.Instruction = onboard.PrescribedEditInstruction return fail("needs_input", "project_dependencies_missing", fmt.Sprintf("dependency setup is missing for %d of the selected configurations", len(setup)), "Run every setup_commands entry (next_command is the first) from the repository root, then run resume_command. This readiness check installed nothing and ran no tests. Preserve the selected framework, CI test commands and scope.") @@ -647,12 +387,12 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work return fail("blocked_environment", "input_snapshot_unavailable", err.Error(), "Restore source input access.") } plan.Fingerprint = before - checkpoint.Fingerprint = before - if err := errors.Join(checkpoint.save(), checkpoint.saveFactsBinding()); err != nil { + attempt.checkpoint.Fingerprint = before + if err := errors.Join(attempt.checkpoint.save(), attempt.checkpoint.saveFactsBinding()); err != nil { return fail("blocked_environment", "checkpoint_unavailable", err.Error(), "Restore private temporary storage.") } started := time.Now() - attemptStarted = started + attempt.started = started logs := io.Discard if options.Format == "verbose" { logs = output @@ -680,7 +420,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if err != nil { return fail("validation_failed", "validation_report_unavailable", fmt.Sprint(errors.Join(runErr, err)), "No current full report exists. Resolve this execution failure; old/check-only evidence cannot certify completion.") } - executed = true + attempt.executed = true plan.CIRuntime = validation.CIRuntime if validation.Selection != nil { t.setPlanTracer(plan, validation.Selection.Version) @@ -716,10 +456,10 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work return fail("validation_failed", "execution_result_conflict", runErr.Error(), "Resolve the execution/report discrepancy before claiming completion.") } progress("verification") - workflow := &WorkflowEvidence{ValidationCommand: recoveryCommand, Plan: plan, Fingerprint: before, Node: t.workflowNodeVersion(), Runtime: languageRuntime, FactsSHA256: factsHash, Changes: result.Changes} - if checkpoint != nil { - workflow.EarlierAttempts = slices.Clone(checkpoint.Failures) - workflow.EarlierVaryingTests = slices.Clone(checkpoint.VaryingTests) + workflow := &WorkflowEvidence{ValidationCommand: attempt.recoveryCommand, Plan: plan, Fingerprint: before, Node: t.workflowNodeVersion(), Runtime: languageRuntime, FactsSHA256: attempt.factsHash, Changes: result.Changes} + if attempt.checkpoint != nil { + workflow.EarlierAttempts = slices.Clone(attempt.checkpoint.Failures) + workflow.EarlierVaryingTests = slices.Clone(attempt.checkpoint.VaryingTests) } workflow.Checks = append(refreshScopeCheck(result.Checks, validation.Scope), t.verifyReportExclusions(ctx)...) after, snapshotErr := workflowFingerprint(ctx, t.repositoryRoot, name) @@ -744,144 +484,7 @@ func (t *Testdrive) Workflow(ctx context.Context, output io.Writer, options Work if finalizeErr != nil && validation.Summary.Status == "COMPLETE LOCALLY" { return fail("blocked_environment", "report_write_failed", finalizeErr.Error(), "Restore report output access before claiming completion.") } - result.Fingerprint = before - result.Checks = workflow.Checks - result.JSONReport = validation.jsonReportPath(t.repositoryRoot) - if validation.HTMLReportCurrent { - result.HTMLReport = validation.htmlReportPath(t.repositoryRoot) - } - result.Instruction = "Validation finished for this source snapshot. Finish any source edits, then run next_command as the final check and report its factual response. Further source edits invalidate completion. ddtest has already checked its owned cleanup and report exclusions; do not repeat cache or package audits." - result.NextAction = "run_command" - resume := result.NextCommand - result.NextCommand = onboard.DDTestCommand("testdrive", "--report") - if options.Format == "json" { - result.NextCommand = append(result.NextCommand, "--format", "json") - } - result.Phase = "complete" - if workflowVerdict(validation) == "selected_passed" { - if err := checkpoint.remove(); err != nil { - return fail("blocked_environment", "checkpoint_cleanup_failed", err.Error(), "Remove only the named owned checkpoint after restoring access.") - } - result.Status = "selected_passed" - if validation.KeptReport != "" { - // testdrive --report renders the kept --all report, not this run. - result.NextAction, result.NextCommand = "report_result", nil - result.FinalResponse = validation.Summary.FinalResponse - result.Instruction = "This run of one command passed. It was written beside the --all report " + validation.KeptReport + ", which it does not replace and which alone can certify onboarding. Report final_response unchanged; do not present this run as certification." - } - return onboard.RenderAgentResult(output, result, options.Format) - } - if workflowVerdict(validation) != "complete_locally" { - result.Instruction = "" - result.NextCommand = nil - // Results that end the agent's turn carry the current report's canonical - // text, so the agent can relay it verbatim (see fail). - turnFinalResponse = validation.Summary.FinalResponse - result.ResumeCommand = append(append([]string{}, resume...), "--resume", checkpoint.RunID) - result.Phase = "validation" - if !workflowChecksPassed(workflow) { - result.Phase = "verification" - } - var reviewCommands, reviewItems []string - for _, configuration := range validation.Configurations { - for _, build := range configuration.Preparation { - if build.Code == "preparation_review_required" && !slices.Contains(reviewCommands, build.Command) { - reviewCommands = append(reviewCommands, build.Command) - item := build.Command - if build.Diagnostic != "" { - item += " (review " + build.Diagnostic + ")" - } - reviewItems = append(reviewItems, item) - } - } - } - if len(reviewCommands) > 0 { - result.Location = strings.Join(reviewCommands, "; ") - result.RequiredInput = []string{"Read " + strings.Join(reviewItems, "; ") + ". The build runs in an isolated copy, never in the checkout. You may run resume_command without asking only when every named step runs repository-local code (package scripts resolving to files in the repository, or bundler configuration with well-known plugins) and nothing downloads or installs (no npx of uncached packages, curl, wget or network installs) or writes outside the repository copy; otherwise ask the user first. The final response reports the opt-in. No new setup facts are required."} - if !slices.Contains(result.ResumeCommand, "--allow-reviewed-builds") { - result.ResumeCommand = append(result.ResumeCommand, "--allow-reviewed-builds") - } - return fail("needs_input", "preparation_review_required", "Preparation was not executed pending review: "+result.Location, "After reviewing the named commands, opt in using resume_command. Preserve the CI commands, scope and assertions; do not invent build facts.") - } - if entry, cause := workflowSetupFailure(validation); entry != nil { - if entry.Cypress != nil && len(entry.Cypress.Start) > 0 && strings.Contains(validation.Error, "cypress startup port") { - result.Location = entry.ID - result.InputTemplate = cypressPortFactsTemplate(t.repositoryRoot, before, *entry) - result.RequiredInput = []string{"supported application/test port override for the existing Cypress startup; keep startup, baseUrl and action wait-on consistent"} - return fail("needs_input", "service_setup_unresolved", strings.Join(validation.Summary.BlockingChecks, "; "), "Use input_template.configurations[0].port_override for an existing application port variable. If the app has none, add a default-preserving variable in its startup source, rerun resume_command once for the new fingerprint, then fill the template outside the checkout and append --facts to resume_command. ddtest applies the local startup/baseUrl/readiness override; do not change CI port defaults or duplicate the service.") - } - result.InputTemplate = setupFactsTemplate(before, *entry) - result.RequiredInput = []string{"correct the named prerequisite or service; supply configuration_id setup facts and run resume_command"} - reason := entry.Command + ": setup did not complete" - if cause != "" { - reason += ": " + cause - } - return fail("needs_input", "configuration_setup_failed", reason, "Tests were not compared for this configuration. Correct only its declared setup; preserve scope and assertions.") - } - // Evidence that passed on a compatible substitute runtime is the user's to accept. - var subBlockers []string - var unaccepted []runtimeSubstitution - for _, sub := range validation.substitutions() { - if !sub.Accepted { - subBlockers = append(subBlockers, sub.blocker()) - unaccepted = append(unaccepted, sub) - } - } - subLocal, subRemedy := substituteChoice(unaccepted) - if len(subBlockers) > 0 && slices.Equal(validation.Summary.BlockingChecks, subBlockers) { - if !slices.Contains(result.ResumeCommand, AcceptSubstituteRuntimeFlag) { - result.ResumeCommand = append(result.ResumeCommand, AcceptSubstituteRuntimeFlag) - } - result.RequiredInput = []string{"User decision: accept the evidence from " + subLocal + " by running resume_command, or " + subRemedy + " and rerun resume_command without " + AcceptSubstituteRuntimeFlag + "."} - return fail("needs_input", "substitute_runtime_unaccepted", strings.Join(subBlockers, "; "), "Every check passed on a compatible substitute runtime. Ask the user which option to take; do not accept it yourself, download a runtime, or change CI, the tracer or the test scope.") - } - reportOnly, actionable := splitTracerOnlyBlockers(validation) - decision := "" - if len(subBlockers) > 0 { - // The substitute is still the user's decision when other blockers remain; - // it must not be lost among them or accepted by the agent. - actionable = slices.DeleteFunc(actionable, func(check string) bool { return slices.Contains(subBlockers, check) }) - acceptFlag = AcceptSubstituteRuntimeFlag - decision = " Separately, ask the user whether to accept the evidence from substitute " + subLocal + ": once they approve, run accept_resume_command instead of resume_command; otherwise " + subRemedy + ". Do not accept it yourself or download a runtime." - } - if len(actionable) == 0 && len(reportOnly) > 0 { - result.RequiredInput = reportOnly - return fail("validation_failed", "validation_incomplete", strings.Join(reportOnly, "; "), "Report these blockers to the user as they are: the tracer did not apply these controls for this framework setup, and no customer change is expected to fix them. Do not change the suite, CI, configuration or tracer to work around them, and do not rerun."+decision) - } - // Every actionable blocker is listed: a capped list hid real checks behind - // lines that only restate them. Multi-line CI scripts are shortened to one - // line; the report keeps them whole. - result.RequiredInput = nil - for _, check := range actionable { - result.RequiredInput = append(result.RequiredInput, onboard.OneLineExcerpt(check)) - } - needs := "Address only these current blockers, then rerun full validation. A failed feature or exclusion check remains failed." - if slices.ContainsFunc(actionable, isScopeBlocker) { - needs = "Address only these current blockers, then rerun full validation; unresolved CI scope blockers are not yours to address. A failed feature or exclusion check remains failed. " + onboard.ScopeBlockerInstruction - if !slices.ContainsFunc(actionable, func(check string) bool { return !isScopeBlocker(check) }) { - needs = "Report these blockers to the user with their options; there is nothing to rerun until the user decides. " + onboard.ScopeBlockerInstruction - } - } - if len(reportOnly) > 0 { - needs += " Report, without trying to fix, these tracer limitations: " + strings.Join(reportOnly, "; ") + "." - } - return fail("validation_failed", "validation_incomplete", strings.Join(result.RequiredInput, "; "), needs+decision) - } - if err := checkpoint.remove(); err != nil { - return fail("blocked_environment", "checkpoint_cleanup_failed", err.Error(), "Restore access to private checkpoint storage.") - } - result.Status = "complete_locally" - return onboard.RenderAgentResult(output, result, options.Format) -} - -// Rejected options, facts or resume IDs describe this invocation, not the -// repository, so they leave an earlier report and its binding untouched. -func workflowInputFailure(code string) bool { - switch code { - case "resume_invalid", "resume_not_found", "facts_invalid", "facts_refresh_required": - return true - } - return false + return attempt.complete(t, validation, workflow, before) } // withWorkflowState carries the workflow's scope, plan, run and selected site diff --git a/internal/testdrive/workflow_attempt.go b/internal/testdrive/workflow_attempt.go new file mode 100644 index 000000000..64c2dcd41 --- /dev/null +++ b/internal/testdrive/workflow_attempt.go @@ -0,0 +1,157 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "errors" + "fmt" + "io" + "slices" + "time" + + "github.com/DataDog/ddtest/internal/onboard" +) + +// workflowAttempt owns one invocation's handoff, checkpoint and retry state. +// Coordinated and standalone validation share its failure policy; standalone +// validation leaves the checkpoint and report-lock ownership unset. +type workflowAttempt struct { + result *onboard.AgentResult + output io.Writer + options WorkflowOptions + framework string + recoveryCommand []string + checkpoint *workflowCheckpoint + factsHash string + ownedLock, executed bool + // Added after the checkpoint rewrites --resume, so this command stays current. + acceptFlag string + finalResponse string + // Failures can read only reports written by this execution attempt. + started time.Time +} + +func (a *workflowAttempt) fail(t *Testdrive, status, code, reason, needs string) error { + result := a.result + + result.Status, result.Code, result.Reason, result.Needs = status, code, reportText(reason), needs + if a.options.Format == "verbose" && result.UndoInstructions != "" { + _, _ = io.WriteString(a.output, result.UndoInstructions) + } + if a.checkpoint != nil { + if a.checkpoint.FactsSHA256 == "" || (code != "facts_invalid" && code != "facts_refresh_required") { + a.checkpoint.Plan = result.Plan + } + var varying []string + if !a.started.IsZero() { + if report, err := readCurrentOnboardingResult(t.repositoryRoot, a.started); err == nil { + varying = varyingTests(report) + } + } + if err := a.checkpoint.recordFailure(result.Fingerprint, a.factsHash, code, result.Reason, varying...); err != nil { + result.Reason += "; checkpoint: " + err.Error() + } + result.RunID = a.checkpoint.RunID + if result.ResumeCommand == nil { + result.ResumeCommand = append(append([]string{}, a.recoveryCommand...), "--resume", a.checkpoint.RunID) + } else { + for i := 0; i+1 < len(result.ResumeCommand); i++ { + if result.ResumeCommand[i] == "--resume" { + result.ResumeCommand[i+1] = a.checkpoint.RunID + } + } + } + } + if a.acceptFlag != "" && len(result.ResumeCommand) > 0 && !slices.Contains(result.ResumeCommand, a.acceptFlag) { + result.AcceptResumeCommand = append(append([]string{}, result.ResumeCommand...), a.acceptFlag) + } + switch status { + case "blocked_environment": + switch code { + case "workflow_busy": + result.NextAction = "wait" + case fileDescriptorsExhaustedCode: + // A transient host limit, not a missing capability: retry as is. + result.NextAction = "run_command" + result.NextCommand = append([]string{}, a.recoveryCommand...) + if len(result.ResumeCommand) > 0 { + result.NextCommand = append([]string{}, result.ResumeCommand...) + } + default: + result.NextAction = "request_capability" + } + case "needs_input": + switch code { + case "scope_unresolved": + result.NextAction = "run_command" + result.NextCommand = onboard.DDTestCommand("onboard", "--framework", a.framework, "--instructions") + result.ResumeCommand, result.RequiredInput = nil, nil + result.Needs = scopeUnresolvedNeeds(result.Needs, a.options.Site) + case "setup_dependencies_missing", "project_dependencies_missing": + result.NextAction = "run_command" + case "ci_edit_required", "report_exclusions_required": + result.NextAction = "apply_source_edit" + case "ci_runtime_required", "ci_setup_confirmation_required": + // Only the user's answer, added to the same command, continues. + result.NextAction = "ask_user" + result.ResumeCommand = nil + default: + result.NextAction = "supply_input" + } + case "unsupported": + if code == "ci_edit_unsupported" && len(result.NextCommand) > 0 { + // The setup to apply comes from next_command (the manual + // instructions); resume_command validates it. + result.NextAction = "run_command" + } else if code == "ci_edit_unsupported" { + result.NextAction = "apply_source_edit" + } else { + result.NextAction = "report_result" + } + default: + result.NextAction = "report_result" + } + if a.ownedLock && !a.executed && !t.checkOnly && !workflowInputFailure(code) { + invalidatePriorWorkflow(t.repositoryRoot, a.framework, *result) + } + switch result.NextAction { + case "report_result", "supply_input", "apply_source_edit", "wait", "ask_user": + result.NextCommand = nil + case "request_capability": + // Without a checkpoint there is nothing to resume; rerun from the start. + if len(result.ResumeCommand) > 0 { + result.NextCommand = append([]string{}, result.ResumeCommand...) + } else { + result.NextCommand = append([]string{}, a.recoveryCommand...) + } + } + if a.checkpoint != nil && a.checkpoint.repeated && !transientWorkflowFailure(code) { + // A deterministic failure with unchanged inputs repeats on every rerun. + result.NextAction = "report_result" + result.NextCommand, result.ResumeCommand = nil, nil + result.Needs += " This attempt failed exactly like the previous one with unchanged inputs, so rerunning cannot progress; stop and report this blocker." + } else if a.checkpoint != nil && a.checkpoint.Attempts >= maxWorkflowAttempts { + result.NextAction = "report_result" + result.NextCommand, result.ResumeCommand = nil, nil + result.Needs += " Three attempts with unchanged inputs made no progress; stop and report this blocker." + } + // A setup repair continues the workflow; a report or a user decision ends + // the turn and must carry the text to relay. + if a.finalResponse != "" && (result.NextAction == "report_result" || code == "substitute_runtime_unaccepted") { + result.FinalResponse = a.finalResponse + } + return errors.Join(fmt.Errorf("%s: %s", code, result.Reason), onboard.RenderAgentResult(a.output, *result, a.options.Format)) +} + +// Rejected options, facts or resume IDs describe this invocation, not the +// repository, so they leave an earlier report and its binding untouched. +func workflowInputFailure(code string) bool { + switch code { + case "resume_invalid", "resume_not_found", "facts_invalid", "facts_refresh_required": + return true + } + return false +} diff --git a/internal/testdrive/workflow_ci.go b/internal/testdrive/workflow_ci.go new file mode 100644 index 000000000..2e231cff5 --- /dev/null +++ b/internal/testdrive/workflow_ci.go @@ -0,0 +1,52 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + + "github.com/DataDog/ddtest/internal/onboard" +) + +// checkCIStatements asks runtime questions before setup confirmations, before +// either coordinated or standalone validation starts execution. +func (a *workflowAttempt) checkCIStatements(ctx context.Context, t *Testdrive) error { + if required := onboard.RequiredCIRuntimes(ctx, t.repositoryRoot, t.language, a.framework); required != nil { + return a.askCIQuestion(t, onboard.CIRuntimeRequiredResult(required)) + } + return a.checkCIConfirmations(t) +} + +// Setup edits can introduce a confirmation, so the coordinator checks again +// after applying them and preparing the selected tracer. +func (a *workflowAttempt) checkCIConfirmations(t *Testdrive) error { + if required := onboard.RequiredCIConfirmations(t.repositoryRoot, t.language, a.framework); required != nil { + return a.askCIQuestion(t, onboard.CIConfirmationRequiredResult(required)) + } + return nil +} + +func (a *workflowAttempt) askCIQuestion(t *Testdrive, asked onboard.AgentResult) error { + a.result.Location, a.result.RequiredInput, a.result.Instruction = asked.Location, asked.RequiredInput, asked.Instruction + return a.fail(t, "needs_input", asked.Code, asked.Reason, asked.Instruction) +} + +// ciPinnedTracer selects only a release shared by every instrumented CI job. +// Callers keep their own preparation policy after this common decision. +func (a *workflowAttempt) ciPinnedTracer(ctx context.Context, t *Testdrive) (string, error) { + pin := "" + for _, job := range t.lang().ciRuntimes(ctx, t.repositoryRoot, a.framework).Jobs { + if candidate := t.lang().ciPinnedTracer(job); candidate != "" { + if pin != "" && pin != candidate { + a.result.Location = job.Workflow + " / " + job.Job + a.result.RequiredInput = []string{"one intended CI tracer release, or separate explicitly scoped runs"} + return "", a.fail(t, "needs_input", "multiple_ci_tracers", "CI selects different tracer releases.", "Preserve every configuration and align the intended tracer selection explicitly.") + } + pin = candidate + } + } + return pin, nil +} diff --git a/internal/testdrive/workflow_ci_test.go b/internal/testdrive/workflow_ci_test.go new file mode 100644 index 000000000..8f3f62a8f --- /dev/null +++ b/internal/testdrive/workflow_ci_test.go @@ -0,0 +1,75 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/onboard" + "github.com/stretchr/testify/require" +) + +type workflowCIInventory struct { + validationLanguage + jobs []onboard.RuntimeFinding +} + +func (l workflowCIInventory) ciRuntimes(context.Context, string, string) onboard.RuntimeCheck { + return onboard.RuntimeCheck{Jobs: l.jobs} +} + +func TestWorkflowCITracerSelectionPreservesEveryJob(t *testing.T) { + for _, language := range []validationLanguage{javascriptLanguage{}, pythonLanguage{}} { + for _, tc := range []struct { + name string + pins []string + want string + }{ + {name: "no pinned setup"}, + {name: "one shared release", pins: []string{"1.2.3", "", "1.2.3"}, want: "1.2.3"}, + {name: "conflicting releases", pins: []string{"1.2.3", "1.2.4"}}, + } { + t.Run(language.name()+"/"+tc.name, func(t *testing.T) { + var jobs []onboard.RuntimeFinding + for _, pin := range tc.pins { + job := onboard.RuntimeFinding{Workflow: "ci.yml", Job: "tests"} + if pin != "" { + job.Tracer, job.TracerRequested = language.tracerPackage()+"@"+pin, pin + } + jobs = append(jobs, job) + } + previous := validationLanguages + validationLanguages = []validationLanguage{workflowCIInventory{validationLanguage: language, jobs: jobs}} + t.Cleanup(func() { validationLanguages = previous }) + root := t.TempDir() + drive := &Testdrive{repositoryRoot: root, language: language.name(), framework: &framework.Vitest{}} + result := onboard.AgentResult{NextCommand: []string{"ddtest", "testdrive", "--all"}} + var output bytes.Buffer + attempt := &workflowAttempt{result: &result, output: &output, options: WorkflowOptions{Format: "json"}, framework: "tests", recoveryCommand: result.NextCommand} + pin, err := attempt.ciPinnedTracer(t.Context(), drive) + if tc.name != "conflicting releases" { + require.NoError(t, err) + require.Equal(t, tc.want, pin) + require.Empty(t, output.String()) + return + } + require.ErrorContains(t, err, "multiple_ci_tracers") + require.Empty(t, pin, "a conflict must not silently choose either release") + var handoff onboard.AgentResult + require.NoError(t, json.Unmarshal(output.Bytes(), &handoff)) + require.Equal(t, "needs_input", handoff.Status) + require.Equal(t, "ci.yml / tests", handoff.Location) + require.Contains(t, handoff.Needs, "Preserve every configuration") + require.Empty(t, handoff.NextCommand) + require.NoFileExists(t, validationPath(root), "a selection conflict must not create validation evidence") + }) + } + } +} diff --git a/internal/testdrive/workflow_completion.go b/internal/testdrive/workflow_completion.go new file mode 100644 index 000000000..7692d3dde --- /dev/null +++ b/internal/testdrive/workflow_completion.go @@ -0,0 +1,148 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/onboard" +) + +// complete turns the verified current report into a final handoff or a bounded +// recovery command. It never starts tests or recomputes a validation verdict. +func (a *workflowAttempt) complete(t *Testdrive, validation validationResult, workflow *WorkflowEvidence, before string) error { + result := a.result + + result.Fingerprint = before + result.Checks = workflow.Checks + result.JSONReport = validation.jsonReportPath(t.repositoryRoot) + if validation.HTMLReportCurrent { + result.HTMLReport = validation.htmlReportPath(t.repositoryRoot) + } + result.Instruction = "Validation finished for this source snapshot. Finish any source edits, then run next_command as the final check and report its factual response. Further source edits invalidate completion. ddtest has already checked its owned cleanup and report exclusions; do not repeat cache or package audits." + result.NextAction = "run_command" + resume := result.NextCommand + result.NextCommand = onboard.DDTestCommand("testdrive", "--report") + if a.options.Format == "json" { + result.NextCommand = append(result.NextCommand, "--format", "json") + } + result.Phase = "complete" + if workflowVerdict(validation) == "selected_passed" { + if err := a.checkpoint.remove(); err != nil { + return a.fail(t, "blocked_environment", "checkpoint_cleanup_failed", err.Error(), "Remove only the named owned checkpoint after restoring access.") + } + result.Status = "selected_passed" + if validation.KeptReport != "" { + // testdrive --report renders the kept --all report, not this run. + result.NextAction, result.NextCommand = "report_result", nil + result.FinalResponse = validation.Summary.FinalResponse + result.Instruction = "This run of one command passed. It was written beside the --all report " + validation.KeptReport + ", which it does not replace and which alone can certify onboarding. Report final_response unchanged; do not present this run as certification." + } + return onboard.RenderAgentResult(a.output, *result, a.options.Format) + } + if workflowVerdict(validation) != "complete_locally" { + result.Instruction = "" + result.NextCommand = nil + // Results that end the agent's turn carry the current report's canonical + // text, so the agent can relay it verbatim (see fail). + a.finalResponse = validation.Summary.FinalResponse + result.ResumeCommand = append(append([]string{}, resume...), "--resume", a.checkpoint.RunID) + result.Phase = "validation" + if !workflowChecksPassed(workflow) { + result.Phase = "verification" + } + var reviewCommands, reviewItems []string + for _, configuration := range validation.Configurations { + for _, build := range configuration.Preparation { + if build.Code == "preparation_review_required" && !slices.Contains(reviewCommands, build.Command) { + reviewCommands = append(reviewCommands, build.Command) + item := build.Command + if build.Diagnostic != "" { + item += " (review " + build.Diagnostic + ")" + } + reviewItems = append(reviewItems, item) + } + } + } + if len(reviewCommands) > 0 { + result.Location = strings.Join(reviewCommands, "; ") + result.RequiredInput = []string{"Read " + strings.Join(reviewItems, "; ") + ". The build runs in an isolated copy, never in the checkout. You may run resume_command without asking only when every named step runs repository-local code (package scripts resolving to files in the repository, or bundler configuration with well-known plugins) and nothing downloads or installs (no npx of uncached packages, curl, wget or network installs) or writes outside the repository copy; otherwise ask the user first. The final response reports the opt-in. No new setup facts are required."} + if !slices.Contains(result.ResumeCommand, "--allow-reviewed-builds") { + result.ResumeCommand = append(result.ResumeCommand, "--allow-reviewed-builds") + } + return a.fail(t, "needs_input", "preparation_review_required", "Preparation was not executed pending review: "+result.Location, "After reviewing the named commands, opt in using resume_command. Preserve the CI commands, scope and assertions; do not invent build facts.") + } + if entry, cause := workflowSetupFailure(validation); entry != nil { + if entry.Cypress != nil && len(entry.Cypress.Start) > 0 && strings.Contains(validation.Error, "cypress startup port") { + result.Location = entry.ID + result.InputTemplate = cypressPortFactsTemplate(t.repositoryRoot, before, *entry) + result.RequiredInput = []string{"supported application/test port override for the existing Cypress startup; keep startup, baseUrl and action wait-on consistent"} + return a.fail(t, "needs_input", "service_setup_unresolved", strings.Join(validation.Summary.BlockingChecks, "; "), "Use input_template.configurations[0].port_override for an existing application port variable. If the app has none, add a default-preserving variable in its startup source, rerun resume_command once for the new fingerprint, then fill the template outside the checkout and append --facts to resume_command. ddtest applies the local startup/baseUrl/readiness override; do not change CI port defaults or duplicate the service.") + } + result.InputTemplate = setupFactsTemplate(before, *entry) + result.RequiredInput = []string{"correct the named prerequisite or service; supply configuration_id setup facts and run resume_command"} + reason := entry.Command + ": setup did not complete" + if cause != "" { + reason += ": " + cause + } + return a.fail(t, "needs_input", "configuration_setup_failed", reason, "Tests were not compared for this configuration. Correct only its declared setup; preserve scope and assertions.") + } + // Evidence that passed on a compatible substitute runtime is the user's to accept. + var subBlockers []string + var unaccepted []runtimeSubstitution + for _, sub := range validation.substitutions() { + if !sub.Accepted { + subBlockers = append(subBlockers, sub.blocker()) + unaccepted = append(unaccepted, sub) + } + } + subLocal, subRemedy := substituteChoice(unaccepted) + if len(subBlockers) > 0 && slices.Equal(validation.Summary.BlockingChecks, subBlockers) { + if !slices.Contains(result.ResumeCommand, AcceptSubstituteRuntimeFlag) { + result.ResumeCommand = append(result.ResumeCommand, AcceptSubstituteRuntimeFlag) + } + result.RequiredInput = []string{"User decision: accept the evidence from " + subLocal + " by running resume_command, or " + subRemedy + " and rerun resume_command without " + AcceptSubstituteRuntimeFlag + "."} + return a.fail(t, "needs_input", "substitute_runtime_unaccepted", strings.Join(subBlockers, "; "), "Every check passed on a compatible substitute runtime. Ask the user which option to take; do not accept it yourself, download a runtime, or change CI, the tracer or the test scope.") + } + reportOnly, actionable := splitTracerOnlyBlockers(validation) + decision := "" + if len(subBlockers) > 0 { + // The substitute is still the user's decision when other blockers remain; + // it must not be lost among them or accepted by the agent. + actionable = slices.DeleteFunc(actionable, func(check string) bool { return slices.Contains(subBlockers, check) }) + a.acceptFlag = AcceptSubstituteRuntimeFlag + decision = " Separately, ask the user whether to accept the evidence from substitute " + subLocal + ": once they approve, run accept_resume_command instead of resume_command; otherwise " + subRemedy + ". Do not accept it yourself or download a runtime." + } + if len(actionable) == 0 && len(reportOnly) > 0 { + result.RequiredInput = reportOnly + return a.fail(t, "validation_failed", "validation_incomplete", strings.Join(reportOnly, "; "), "Report these blockers to the user as they are: the tracer did not apply these controls for this framework setup, and no customer change is expected to fix them. Do not change the suite, CI, configuration or tracer to work around them, and do not rerun."+decision) + } + // Every actionable blocker is listed: a capped list hid real checks behind + // lines that only restate them. Multi-line CI scripts are shortened to one + // line; the report keeps them whole. + result.RequiredInput = nil + for _, check := range actionable { + result.RequiredInput = append(result.RequiredInput, onboard.OneLineExcerpt(check)) + } + needs := "Address only these current blockers, then rerun full validation. A failed feature or exclusion check remains failed." + if slices.ContainsFunc(actionable, isScopeBlocker) { + needs = "Address only these current blockers, then rerun full validation; unresolved CI scope blockers are not yours to address. A failed feature or exclusion check remains failed. " + onboard.ScopeBlockerInstruction + if !slices.ContainsFunc(actionable, func(check string) bool { return !isScopeBlocker(check) }) { + needs = "Report these blockers to the user with their options; there is nothing to rerun until the user decides. " + onboard.ScopeBlockerInstruction + } + } + if len(reportOnly) > 0 { + needs += " Report, without trying to fix, these tracer limitations: " + strings.Join(reportOnly, "; ") + "." + } + return a.fail(t, "validation_failed", "validation_incomplete", strings.Join(result.RequiredInput, "; "), needs+decision) + } + if err := a.checkpoint.remove(); err != nil { + return a.fail(t, "blocked_environment", "checkpoint_cleanup_failed", err.Error(), "Restore access to private checkpoint storage.") + } + result.Status = "complete_locally" + return onboard.RenderAgentResult(a.output, *result, a.options.Format) +} diff --git a/internal/testdrive/workflow_setup.go b/internal/testdrive/workflow_setup.go new file mode 100644 index 000000000..1a7a712de --- /dev/null +++ b/internal/testdrive/workflow_setup.go @@ -0,0 +1,173 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/url" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" +) + +// acceptSetupFacts binds only validated additions to this checkpoint's inputs. +func (a *workflowAttempt) acceptSetupFacts(t *Testdrive, fingerprint string, scope *onboard.ValidationScope) error { + if a.options.FactsFile == "" { + return nil + } + result := a.result + var err error + + factsFingerprint := fingerprint + // Built before facts touch the scope: rejected facts are rewritten from it. + template := scopeFactsTemplate(fingerprint, *scope) + data, readErr := readSetupFacts(t.repositoryRoot, a.options.FactsFile) + a.factsHash = fmt.Sprintf("%x", sha256.Sum256(data)) + if readErr == nil && a.options.Resume == "" { + a.checkpoint.restoreFactsBinding(fingerprint, a.factsHash) + } + if readErr == nil && a.checkpoint.Fingerprint == fingerprint && a.checkpoint.FactsSHA256 == a.factsHash && a.checkpoint.Plan != nil && a.checkpoint.Engine == engineIdentity() { + // These exact facts already passed validation for the pre-edit inputs. + // Reuse only their additions, bound to the unchanged owned edit result, + // whose plan may still hold the pre-edit commands. + err = reattachSetupFacts(scope, a.checkpoint.Plan.Scope, true) + factsFingerprint = a.checkpoint.FactsFingerprint + } else { + if readErr != nil { + err = readErr + } else { + _, err = applySetupFactsData(t.repositoryRoot, data, fingerprint, scope) + } + } + + if err != nil && a.checkpoint.FactsDependencyHandoff && readErr == nil && a.checkpoint.FactsSHA256 == a.factsHash && a.checkpoint.Engine == engineIdentity() { + var refresh onboard.SetupFacts + if json.Unmarshal(data, &refresh) == nil { + refresh.Fingerprint = fingerprint + result.InputTemplate = &refresh + result.Location = a.options.FactsFile + result.RequiredInput = []string{"Review the unchanged setup entries against the post-install checkout, then replace the facts file with input_template and run resume_command."} + return a.fail(t, "needs_input", "facts_refresh_required", "The declared dependency-install handoff changed repository inputs; the old facts cannot certify the new snapshot.", "Explicitly refresh facts for the returned current fingerprint; do not retry unchanged facts or bypass their binding.") + } + } + if err != nil { + result.Location = a.options.FactsFile + result.InputTemplate = template + result.RequiredInput = []string{"schema_version=2, current input_fingerprint, configurations with configuration_id and build/service setup"} + return a.fail(t, "needs_input", "facts_invalid", err.Error(), "Rewrite the facts file outside the checkout from input_template for the current input_fingerprint, supplying only the missing prerequisite facts, then run resume_command. No discovery or validation assertion can be overridden.") + } + if a.checkpoint.FactsSHA256 != a.factsHash || a.checkpoint.Fingerprint != fingerprint || a.checkpoint.Engine != engineIdentity() { + a.checkpoint.Tracer = "" // Replacement setup invalidates cached preparation. + } + a.checkpoint.FactsSHA256 = a.factsHash + a.checkpoint.Fingerprint = fingerprint + a.checkpoint.Engine = engineIdentity() + a.checkpoint.FactsFingerprint = factsFingerprint + + return nil +} + +func (a *workflowAttempt) failTracerSelection(t *Testdrive, problem error) error { + result := a.result + + var metadata *platform.MetadataError + if errors.As(problem, &metadata) { + result.Capabilities = []string{workflowMetadataCapability(metadata)} + return a.fail(t, "blocked_environment", metadata.Code, problem.Error(), "Grant/fix only the named capability and rerun next_command. No tracer version was guessed and no source edits were made.") + } + var transport *url.Error + if errors.As(problem, &transport) { + address, _ := url.Parse(transport.URL) + capability := "CI metadata network access" + if address != nil { + capability = "network access to " + address.Hostname() + } + result.Capabilities = []string{capability} + return a.fail(t, "blocked_environment", "metadata_network", problem.Error(), "Restore only the named metadata network capability, then run resume_command. Do not change runtime, tracer or test scope to work around a network failure.") + } + var majors *ciTracerMajors + if errors.As(problem, &majors) { + // Resume with the release ddtest recommends rather than the default. + if majors.Recommended != "" && !a.options.ExplicitVersion && a.checkpoint != nil { + result.ResumeCommand = append(append([]string{}, a.recoveryCommand...), "--tracer-version", majors.Recommended, "--resume", a.checkpoint.RunID) + } + if len(majors.Exclusions) > 0 { + result.RequiredInput = []string{"the user's choice: one --tracer-version every instrumented CI row supports, or approval to opt these rows out: " + strings.Join(majors.Exclusions, "; ")} + return a.fail(t, "needs_input", "preflight_unresolved", problem.Error(), "Ask the user before excluding any CI row. If they approve, make exactly the listed DD_CIVISIBILITY_ENABLED edits and run resume_command; otherwise rerun with one --tracer-version every instrumented row supports. Never weaken feature checks.") + } + result.RequiredInput = []string{"one --tracer-version every instrumented CI row supports"} + needs := "No opt-out edit applies to these CI rows; do not exclude or rewrite them. Report them as unvalidated blockers unless one --tracer-version supports every instrumented row." + if majors.Recommended != "" { + needs = "Run resume_command, which selects dd-trace@" + majors.Recommended + ". Rows it cannot run stay unvalidated blockers to report; do not exclude or rewrite CI rows." + } + return a.fail(t, "needs_input", "preflight_unresolved", problem.Error(), needs) + } + result.RequiredInput = []string{"the named runtime/configuration incompatibility"} + return a.fail(t, "needs_input", "preflight_unresolved", problem.Error(), "Resolve this exact mismatch without weakening feature checks. Do not exclude or rewrite CI rows to get past it; if it can't be resolved, report it as a blocker.") +} + +func (a *workflowAttempt) failSetupEdits(t *Testdrive, err error, selected, fingerprint string) error { + result := a.result + + var dependencies *onboard.SetupDependencyGap + if errors.As(err, &dependencies) { + a.checkpoint.FactsDependencyHandoff = true + a.checkpoint.DependencyPackages = dependencies.Packages + a.checkpoint.Engine = engineIdentity() + a.checkpoint.FactsSHA256 = a.factsHash + a.checkpoint.Fingerprint = fingerprint + result.Location = "package.json / devDependencies" + result.NextCommand = dependencies.Command + result.ResumeCommand = append(append([]string{}, a.recoveryCommand...), "--resume", a.checkpoint.RunID) + result.RequiredInput = []string{"Run next_command once, as given and without asking, to update the manifest and lockfile, then resume_command. Preserve the selected framework and CI test commands."} + result.Instruction = onboard.PrescribedEditInstruction + return a.fail(t, "needs_input", "setup_dependencies_missing", err.Error(), "The recognized coverage recipe needs only these dependencies; installing them is a ddtest-prescribed edit, listed with its revert in the final report. No CI edits have been made.") + } + var dirty *onboard.DirtyWorkflowGap + if errors.As(err, &dirty) { + result.Location = strings.Join(dirty.Files, ", ") + result.RequiredInput = []string{"Review setup_diff against the existing target-file edits. With approval to preserve and extend them, run resume_command with --allow-dirty; do not switch to testdrive or change scope."} + result.ResumeCommand = append(append([]string{}, a.recoveryCommand...), "--allow-dirty", "--resume", a.checkpoint.RunID) + return a.fail(t, "needs_input", "dirty_workflow", err.Error(), "Commit or stash the listed target files, then rerun; add --allow-dirty only when the user approves extending those uncommitted edits.") + } + var runtimeGap *onboard.NodeRuntimeGap + if errors.As(err, &runtimeGap) { + result.Location = runtimeGap.Location + result.RequiredInput = []string{"A CI Node version that satisfies " + runtimeGap.Requirement + ", or an older tracer selected with --tracer-version"} + return a.fail(t, "needs_input", "ci_runtime_incompatible", err.Error(), "Upgrade the CI Node version, or select an older tracer with --tracer-version, then rerun. No CI hook needs to be written by hand.") + } + var editGap *onboard.EditThenApplyGap + if errors.As(err, &editGap) { + // One source edit, then onboard --apply writes the setup itself: + // validating before that would check CI without the setup. + result.Location = editGap.Location + result.RequiredInput = []string{"Only the source edit reason names, at " + editGap.Location + " (and a dependency it names, through its install command); then run resume_command, which adds the Datadog CI setup and validates."} + result.ResumeCommand = append([]string{}, a.recoveryCommand...) + if !slices.Contains(result.ResumeCommand, "--allow-dirty") { + // It extends the uncommitted edit just made. + result.ResumeCommand = append(result.ResumeCommand, "--allow-dirty") + } + result.NextCommand = nil + return a.fail(t, "needs_input", "ci_edit_required", err.Error(), "Make only the edit reason names, then run resume_command: onboard --apply adds the Datadog CI setup and validates. Do not run testdrive first."+manualSiteSentence(a.options.Site)) + } + var gap *onboard.SetupGap + if errors.As(err, &gap) { + result.Location = gap.Location + result.RequiredInput = []string{"CI tracer initialization at " + gap.Location} + result.ResumeCommand = t.workflowValidationResume(a.recoveryCommand[0], a.framework, selected, a.options) + a.checkpoint.Apply = false + a.checkpoint.All = true + result.ResumeCommand = append(result.ResumeCommand, "--resume", a.checkpoint.RunID) + result.NextCommand = nil + return a.fail(t, "unsupported", "ci_edit_unsupported", err.Error(), "Integrate only this CI hook, then run resume_command. Do not repeatedly retry the automatic editor or drop configurations."+manualSiteSentence(a.options.Site)) + } + return a.fail(t, "blocked_environment", "ci_edit_unavailable", err.Error(), "Restore filesystem access. Completed source edits are listed in changes; do not overwrite customer edits.") +} From c764d46b95508932c6b51fe5260e5389775caf3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 5 Nov 2026 21:20:02 +0100 Subject: [PATCH 4/5] Extract shared validation engine from framework adapters --- internal/onboard/README.md | 3 + internal/testdrive/java_language.go | 4 +- internal/testdrive/javascript_language.go | 52 +-- internal/testdrive/jest.go | 405 ------------------ internal/testdrive/jest_preflight.go | 100 ----- internal/testdrive/language.go | 45 -- internal/testdrive/native_results.go | 327 ++++++++++++++ internal/testdrive/paired_validation.go | 152 +++++++ internal/testdrive/preflight.go | 113 +++++ internal/testdrive/preflight_dispatch_test.go | 47 +- internal/testdrive/python_language.go | 9 +- internal/testdrive/ruby_language.go | 4 +- 12 files changed, 665 insertions(+), 596 deletions(-) create mode 100644 internal/testdrive/native_results.go create mode 100644 internal/testdrive/paired_validation.go create mode 100644 internal/testdrive/preflight.go diff --git a/internal/onboard/README.md b/internal/onboard/README.md index 0a8f6b8f2..c56241b2d 100644 --- a/internal/onboard/README.md +++ b/internal/onboard/README.md @@ -22,6 +22,9 @@ Start at the boundary that owns the behavior you need to change: | Literal command recognition without executing repository code | `internal/testcommand` | | Setup plans and bounded source edits | `internal/onboard/execution_plan.go`, `apply*.go` | | Validation preparation and execution | `internal/testdrive/testdrive.go` | +| Shared paired-execution engine and framework adapters | `internal/testdrive/paired_validation.go` | +| Shared preflight evidence and CI configuration checks | `internal/testdrive/preflight.go` | +| Native result model, outcome comparison and telemetry matching | `internal/testdrive/native_results.go` | | Coordinated setup and validation | `internal/testdrive/workflow.go` | | Attempt state, failure policy and retry handoffs | `internal/testdrive/workflow_attempt.go` | | Shared CI questions and pinned tracer selection | `internal/testdrive/workflow_ci.go` | diff --git a/internal/testdrive/java_language.go b/internal/testdrive/java_language.go index d24f2b1bc..dcc42a517 100644 --- a/internal/testdrive/java_language.go +++ b/internal/testdrive/java_language.go @@ -248,9 +248,7 @@ func (javaLanguage) pairedFrameworks() map[string]pairedFramework { features: func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, agent string, baseline validationRun, result *validationResult) error { return t.runJavaFeatures(ctx, output, session, agent, baseline, result) }, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkJavaPreflight - }, + preflight: (*Testdrive).checkJavaPreflight, telemetry: verifyJavaTelemetry, // The agent reports no source file for some frameworks, such as // Cucumber, so the probe is matched by module, class and name. diff --git a/internal/testdrive/javascript_language.go b/internal/testdrive/javascript_language.go index bf8a898e9..4cd1979e4 100644 --- a/internal/testdrive/javascript_language.go +++ b/internal/testdrive/javascript_language.go @@ -95,51 +95,37 @@ func (javascriptLanguage) pairedFrameworks() map[string]pairedFramework { features: func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, preload string, _ validationRun, result *validationResult) error { return t.runJestFeatures(ctx, output, session, preload, result) }, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkJestPreflight - }, + preflight: (*Testdrive).checkJestPreflight, }, "vitest": { - run: (*Testdrive).runVitest, - features: (*Testdrive).runVitestFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkVitestPreflight - }, + run: (*Testdrive).runVitest, + features: (*Testdrive).runVitestFeatures, + preflight: (*Testdrive).checkVitestPreflight, }, "mocha": { - run: (*Testdrive).runMocha, - features: (*Testdrive).runMochaFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkMochaPreflight - }, + run: (*Testdrive).runMocha, + features: (*Testdrive).runMochaFeatures, + preflight: (*Testdrive).checkMochaPreflight, }, "playwright": { - run: (*Testdrive).runPlaywright, - features: (*Testdrive).runPlaywrightFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkPlaywrightPreflight - }, + run: (*Testdrive).runPlaywright, + features: (*Testdrive).runPlaywrightFeatures, + preflight: (*Testdrive).checkPlaywrightPreflight, }, "cypress": { - run: (*Testdrive).runCypress, - features: (*Testdrive).runCypressFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkCypressPreflight - }, + run: (*Testdrive).runCypress, + features: (*Testdrive).runCypressFeatures, + preflight: (*Testdrive).checkCypressPreflight, }, "cucumber": { - run: (*Testdrive).runCucumber, - features: (*Testdrive).runCucumberFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkCucumberPreflight - }, + run: (*Testdrive).runCucumber, + features: (*Testdrive).runCucumberFeatures, + preflight: (*Testdrive).checkCucumberPreflight, }, "webdriverio": { - run: (*Testdrive).runWebdriverIO, - features: (*Testdrive).runWebdriverIOFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkWebdriverIOPreflight - }, + run: (*Testdrive).runWebdriverIO, + features: (*Testdrive).runWebdriverIOFeatures, + preflight: (*Testdrive).checkWebdriverIOPreflight, }, } } diff --git a/internal/testdrive/jest.go b/internal/testdrive/jest.go index c3cb456c3..ec1c3dccf 100644 --- a/internal/testdrive/jest.go +++ b/internal/testdrive/jest.go @@ -13,40 +13,14 @@ import ( "io" "os" "path/filepath" - "regexp" "slices" "strings" "github.com/DataDog/ddtest/internal/framework" - "github.com/DataDog/ddtest/internal/testdrive/intake" "github.com/DataDog/ddtest/internal/testdrive/workspace" "github.com/kballard/go-shellquote" ) -type jestTest struct { - WebdriverIOCapability string `json:"webdriverio_capability,omitempty"` - WebdriverIOTestID string `json:"webdriverio_test_id,omitempty"` - WebdriverIOBrowser string `json:"webdriverio_browser,omitempty"` - CucumberAttempts []string `json:"-"` - WebdriverIOAttempts []string `json:"-"` - CucumberLine int `json:"-"` - CypressAttempts []string `json:"-"` - Playwright *playwrightIdentity `json:"playwright,omitempty"` - // NativeID is the runner's own unique example identity, reported in the - // tracer's parameters metadata as scoped_id. - NativeID string `json:"native_id,omitempty"` - // telemetryClass qualifies telemetryName with its test class, which the - // tracer reports as the suite. - telemetryClass string - // probe marks the controlled probe when its native name is not probeName. - probe bool - telemetryName *string - File string `json:"file"` - Name string `json:"name"` - Status string `json:"status"` - Failure string `json:"failure,omitempty"` -} - type jestResults struct { TestResults []struct { Name string `json:"name"` @@ -62,42 +36,6 @@ type jestResults struct { } `json:"testResults"` } -var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) - -// Object addresses differ between processes. Only addresses inside a -// recognizable object representation are replaced: Ruby's # -// (and #), Python's -// and Java's com.acme.Widget@1b6d3586, an array's [Ljava.lang.String;@1b6d3586 -// or [I@1b6d3586, or a lambda's Test$$Lambda$1352/0x000000980176c488@4f3b5c60, -// whose number and hidden class address differ too. Bare hexadecimal values -// are data and stay, so differing values still differ. -var ( - inspectedObjectAddress = regexp.MustCompile(`(#<[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*:)0x[0-9a-fA-F]{6,}`) - reprObjectAddress = regexp.MustCompile(`(<[A-Za-z_][A-Za-z0-9_.]* object at )0x[0-9a-fA-F]{6,}(>)`) - lambdaObject = regexp.MustCompile(`\$\$Lambda(?:\$[0-9]+)?(?:/0x[0-9a-f]+)?@[0-9a-f]{1,8}\b`) - objectHash = regexp.MustCompile(`(\b[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)+|\[+L[A-Za-z_$][A-Za-z0-9_$.]*;|\[+[ZBCSIJFD])@[0-9a-f]{1,8}\b`) -) - -// withoutObjectAddresses replaces object addresses in a failure message, -// so the same failure compares equal across runs. -func withoutObjectAddresses(message string) string { - message = inspectedObjectAddress.ReplaceAllString(message, "${1}0x…") - message = reprObjectAddress.ReplaceAllString(message, "${1}0x…${2}") - message = lambdaObject.ReplaceAllString(message, "$$$$Lambda@…") - return objectHash.ReplaceAllString(message, "$1@…") -} - -func failureSignature(message string) string { - var lines []string - for _, line := range strings.Split(withoutObjectAddresses(ansiEscape.ReplaceAllString(message, "")), "\n") { - line = strings.TrimSpace(line) - if line != "" && !strings.HasPrefix(line, "at ") { - lines = append(lines, line) - } - } - return strings.Join(lines, "\n") -} - func readJestResults(path string, run *validationRun) { readRunnerResults(path, "Jest", run) } func readRunnerResults(path, label string, run *validationRun) { @@ -292,353 +230,10 @@ func (t *Testdrive) runJest(ctx context.Context, output io.Writer, session *work // Preserve only the bounded tail where Jest normally prints its final errors, // including coverage failures which do not appear in its JSON test results. -func commandDiagnostic(output []byte) string { - text := []rune(failureSignature(string(output))) - const limit = diagnosticLimit - if len(text) > limit { - const prefix = "[truncated] …" - return prefix + string(text[len(text)-(limit-len([]rune(prefix))):]) - } - return string(text) -} - -func configureScenarioEnvironment(env map[string]string, feature string) { - // Coverage requires ITR negotiation; tests_skipping is separately disabled by - // the reporting-only settings response. No skippable tests are returned there. - env["DD_CIVISIBILITY_ITR_ENABLED"] = "true" - env["DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"] = fmt.Sprint(feature == "early-flake-detection") - env["DD_TEST_EARLY_FLAKE_DETECTION_RETRY_COUNT"] = fmt.Sprint(intake.ScenarioRetryCount) - env["DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"] = fmt.Sprint(feature == "auto-retries") - env["DD_CIVISIBILITY_FLAKY_RETRY_COUNT"] = fmt.Sprint(intake.ScenarioRetryCount) - env["DD_TEST_MANAGEMENT_ENABLED"] = fmt.Sprint(feature == "quarantine" || feature == "disabled" || feature == "attempt-to-fix") - env["DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"] = fmt.Sprint(intake.ScenarioRetryCount) -} - -func executedTests(run validationRun) int { - count := 0 - for _, test := range run.Tests { - if test.Status == "passed" || test.Status == "failed" || (run.framework == "cucumber" && (test.Status == "pending" || test.Status == "undefined")) { - count++ - } - } - return count -} - -func outcomeKeys(run validationRun) []string { - keys := []string{fmt.Sprintf("exit=%d", run.ExitCode), "result=" + run.ResultError} - keys = append(keys, run.SuiteErrors...) - keys = append(keys, run.SuiteFailures...) - for _, test := range run.Tests { - scoped := "" - if test.NativeID != "" { - scoped = " [scoped_id: " + test.NativeID + "]" - } - keys = append(keys, repositoryTestFile(run, test.File)+" › "+test.Name+scoped+cucumberLocation(test)+playwrightLabel(test)+webdriverIOOutcome(test)+": "+test.Status+"\n"+test.Failure+playwrightOutcome(test)+cypressOutcome(test)+cucumberOutcome(test)+webdriverIOAttemptOutcome(test)) - } - for i := range keys { - keys[i] = normalizeOwnedTemporaryPath(run, keys[i]) - } - slices.Sort(keys) - return keys -} - -// pairTimeout explains a paired comparison that testdrive's --timeout cut short. -func pairTimeout(baseline, instrumented validationRun) string { - if baseline.Timeout != "" { - return baseline.Timeout + "." - } - if instrumented.Timeout != "" { - return instrumented.Timeout + ", while the uninstrumented run finished." - } - return "" -} - func compareJest(baseline, instrumented validationRun) verdict { return compareRuns(baseline, instrumented, verifyJestTelemetry) } -// compareRuns compares an uninstrumented and a reporting-only run of the same -// suite, then matches the reporting run's telemetry with telemetry. -func compareRuns(baseline, instrumented validationRun, telemetry func(validationRun) verdict) verdict { - if reason := pairTimeout(baseline, instrumented); reason != "" { - return verdict{Status: "inconclusive", Code: "command_timeout", Reason: reason} - } - if baseline.ResultError != "" { - return verdict{Status: "inconclusive", Reason: "The uninstrumented run did not produce usable results: " + reportText(baseline.ResultError) + "."} - } - if executedTests(baseline) == 0 { - return verdict{Status: "inconclusive", Reason: "The uninstrumented run did not produce results for any executed tests; check project setup using the recorded baseline command."} - } - if baseline.Facts.TestEventCount > 0 { - return verdict{Status: "inconclusive", Reason: "The baseline emitted test telemetry despite instrumentation being disabled; a clean comparison was not established."} - } - left, right := outcomeKeys(baseline), outcomeKeys(instrumented) - if !slices.Equal(left, right) { - differences := []string{} - counts := make(map[string]int) - for _, key := range left { - counts[key]++ - } - for _, key := range right { - counts[key]-- - } - for key, count := range counts { - if count == 0 { - continue - } - label, _, _ := strings.Cut(key, "\n") - side := "Without instrumentation only" - if count < 0 { - side = "With instrumentation only" - } - differences = append(differences, side+": "+label) - } - slices.Sort(differences) - return verdict{Status: "suspected regression", Reason: "Uninstrumented and reporting-only results differ.", Differences: differences} - } - if telemetry := telemetry(instrumented); telemetry.Status != "" { - return telemetry - } - return verdict{Status: "compatible", Reason: "No observed behavioral regression: test identities, outcomes, failure details, and exit code match. Existing test failures are acceptable; reporting telemetry was verified."} -} - func verifyJestTelemetry(run validationRun) verdict { return verifyNativeTelemetry(run, false) } - -func verifyNativeTelemetry(run validationRun, includeSkipped bool) verdict { - if run.framework == "webdriverio" { - return verifyWebdriverIOTelemetry(run) - } - if run.framework == "cucumber" { - return verifyCucumberTelemetry(run) - } - if run.framework == "playwright" { - return verifyPlaywrightTelemetry(run) - } - if run.framework == "cypress" { - return verifyCypressTelemetry(run) - } - if run.Facts.TestEventCount == 0 { - return verdict{Status: "inconclusive", Reason: "The instrumented run sent no test events; matching test outcomes do not prove instrumentation worked."} - } - if len(run.Facts.ConfigurationErrors) > 0 { - return verdict{Status: "inconclusive", Reason: "The tracer reported configuration errors: " + strings.Join(run.Facts.ConfigurationErrors, ", ")} - } - type identity struct{ file, name, status string } - expected := make(map[identity]int) - scopedExamples := make(map[string]bool) - classes := make(map[string][]string) - for _, test := range run.Tests { - status := map[string]string{"passed": "pass", "failed": "fail"}[test.Status] - if includeSkipped && test.Status == "skipped" { - status = "skip" - } - if status != "" { - name := test.Name - if test.telemetryName != nil { - name = *test.telemetryName - } - if test.telemetryClass != "" { - key := test.File + "\x00" + name - if !slices.Contains(classes[key], test.telemetryClass) { - classes[key] = append(classes[key], test.telemetryClass) - } - name = test.telemetryClass + "\x00" + name - } - if test.NativeID != "" { - name = "scoped_id:" + test.NativeID - scopedExamples[test.File+"\x00"+name] = true - } - expected[identity{test.File, name, status}]++ - } - } - observed := make(map[identity]int) - var differences []string - for _, test := range run.Facts.Tests { - // The tracer reports source files relative to the Git checkout, and - // suite names relative to the test process directory. - file, base := test.SourceFile, run.checkoutRoot - if file == "" { - file, base = test.Suite, run.root - } - if base == "" { - base = run.root - } - if !filepath.IsAbs(file) { - file = filepath.Join(base, file) - } - file = filepath.Clean(file) - name := test.Name - if candidates := classes[file+"\x00"+name]; len(candidates) > 0 { - var matches []string - for _, class := range candidates { - if test.Suite == class || strings.HasPrefix(test.Suite, class+" at ") { - matches = append(matches, class) - } - } - // Missing or ambiguous class evidence must not match by method alone. - if len(matches) == 1 { - name = matches[0] + "\x00" + name - } - } - if len(scopedExamples) > 0 { - var parameters struct { - Metadata struct { - ScopedID string `json:"scoped_id"` - } - } - if json.Unmarshal([]byte(test.Parameters), ¶meters) == nil { - id := "scoped_id:" + parameters.Metadata.ScopedID - if scopedExamples[file+"\x00"+id] { - name = id - } - } - } - observe := func(attempt intake.TestRun) { - if attempt.Status == "pass" || attempt.Status == "fail" || includeSkipped && attempt.Status == "skip" { - observed[identity{file, name, attempt.Status}]++ - } - } - // A span sent twice is one execution; counting it again could hide a - // missing test that shares its name, such as another outline example. - spans := map[uint64]int{} - attempts := make([]intake.TestRun, 0, len(test.Attempts)) - for _, attempt := range test.Attempts { - if attempt.SpanID != 0 { - spans[attempt.SpanID]++ - if spans[attempt.SpanID] > 1 { - continue - } - } - attempts = append(attempts, attempt) - } - for _, count := range spans { - if count > 1 { - differences = append(differences, fmt.Sprintf("Duplicate telemetry: file %q, test %q, one span reported %d times", file, test.Name, count)) - } - } - var final *intake.TestRun - for i, attempt := range attempts { - if attempt.Retry && attempt.RetryReason != "external" && attempt.RetryReason != "" { - return verdict{Status: "inconclusive", Code: "unexpected_retry", Reason: "Unexpected tracer-controlled retry telemetry in the reporting-only run."} - } - if !attempt.Retry && final != nil { - observe(*final) - } - final = &attempts[i] - } - if final != nil { - observe(*final) - } - } - for key, count := range expected { - if count > observed[key] { - differences = append(differences, fmt.Sprintf("Missing telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-observed[key])) - } - } - for key, count := range observed { - if count > expected[key] { - differences = append(differences, fmt.Sprintf("Unexpected telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-expected[key])) - } - } - if len(differences) > 0 { - slices.Sort(differences) - return verdict{Status: "inconclusive", Reason: "Reported telemetry does not match the executed tests.", Differences: differences} - } - return verdict{} -} - -func (t *Testdrive) runFrameworkValidation(ctx context.Context, output io.Writer, session *workspace.Session, preload string, result *validationResult) error { - adapter, ok := pairedFrameworkNamed(t.framework.Name()) - if !ok { - return fmt.Errorf("paired validation is not supported for %s", t.framework.Name()) - } - run := func(ctx context.Context, output io.Writer, session *workspace.Session, preload string, spec runSpec) (validationRun, error) { - return adapter.run(t, ctx, output, session, preload, spec) - } - runFramework := run - run = func(ctx context.Context, output io.Writer, session *workspace.Session, preload string, spec runSpec) (validationRun, error) { - value, err := runFramework(ctx, output, session, preload, spec) - return markTimeout(value), err - } - baseline, err := run(ctx, output, session, preload, runSpec{Name: "baseline", Role: roleBaseline, Instrumented: false}) - result.Runs = append(result.Runs, baseline.summary()) - if err != nil { - return err - } - instrumented, err := run(ctx, output, session, preload, runSpec{Name: "reporting-only", Role: roleReporting, Instrumented: true}) - result.Runs = append(result.Runs, instrumented.summary()) - if err != nil { - return err - } - if err := t.writeHTMLReport(output, result, instrumented); err != nil { - return err - } - telemetry := adapter.telemetry - if telemetry == nil { - telemetry = verifyJestTelemetry - } - result.Compatibility = compareRuns(baseline, instrumented, telemetry) - if result.Compatibility.Status == "suspected regression" { - baselineAgain, err := run(ctx, output, session, preload, runSpec{Name: "baseline-repeat", Role: roleRepeat, Instrumented: false}) - result.Runs = append(result.Runs, baselineAgain.summary()) - if err != nil { - return err - } - instrumentedAgain, err := run(ctx, output, session, preload, runSpec{Name: "reporting-only-repeat", Role: roleRepeat, Instrumented: true}) - result.Runs = append(result.Runs, instrumentedAgain.summary()) - if err != nil { - return err - } - if err := t.writeHTMLReport(output, result, instrumentedAgain); err != nil { - return err - } - result.Compatibility = compareRepeatedRuns(baseline, instrumented, baselineAgain, instrumentedAgain, telemetry) - } - // Probes would only repeat the hang; name the timeout once instead. - if reason := pairTimeout(baseline, instrumented); reason != "" { - unavailableFeatures(result, reason) - return nil - } - if compatibility, feature, ok := missingNativeResults(baseline); ok { - result.Features = []featureResult{{Name: "all", Status: "not exercised", Reason: feature}} - result.Compatibility = compatibility - return nil - } - return adapter.features(t, ctx, output, session, preload, baseline, result) -} - -// Only the temporary directory created for this run is interchangeable. Keep -// file suffixes and arbitrary customer paths/names intact, including failures. -// repositoryTestFile shows a native test file relative to the checkout the run -// used, so differences name repository paths, not a temporary copy. -func repositoryTestFile(run validationRun, file string) string { - if run.root == "" || !filepath.IsAbs(file) { - return file - } - if relative, err := filepath.Rel(run.root, file); err == nil && filepath.IsLocal(relative) { - return filepath.ToSlash(relative) - } - return file -} - -func normalizeOwnedTemporaryPath(run validationRun, value string) string { - if run.temporaryRoot == "" { - return value - } - return strings.ReplaceAll(value, run.temporaryRoot+string(filepath.Separator), "/") -} - -// missingNativeResults explains a baseline without test results. A failed -// command is a startup/setup problem; a successful one that ddtest can't read -// is a result-reading problem and must not send agents to change the suite. -func missingNativeResults(baseline validationRun) (verdict, string, bool) { - if len(baseline.Tests) > 0 || (baseline.ResultError == "" && baseline.ExitCode == 0) { - return verdict{}, "", false - } - if baseline.ExitCode == 0 { - return verdict{Status: "inconclusive", Code: "native_results_unreadable", Reason: reportText(baseline.ResultError) + ". The native suite exited 0, but ddtest could not read its results, so features were not validated. If this command runs the tests, this is a ddtest result-reading problem, not a suite failure: report it with the ddtest version instead of changing the repository."}, "ddtest could not read the native results of a successful suite run, so features were not validated.", true - } - return verdict{Status: "inconclusive", Code: "native_startup_failed", Reason: "Native execution failed before any test results. Resolve the baseline startup/setup diagnostic first: " + reportText(baseline.Diagnostic+" "+baseline.ResultError)}, "The native suite did not produce test results. Resolve its startup/setup diagnostic before feature or coverage validation.", true -} diff --git a/internal/testdrive/jest_preflight.go b/internal/testdrive/jest_preflight.go index fc7599615..c4cd7ea37 100644 --- a/internal/testdrive/jest_preflight.go +++ b/internal/testdrive/jest_preflight.go @@ -19,7 +19,6 @@ import ( "strings" "github.com/DataDog/ddtest/internal/framework" - "github.com/DataDog/ddtest/internal/git" "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/testdrive/workspace" @@ -39,26 +38,6 @@ type jestProject struct { ExtensionsToTreatAsEsm []string `json:"extensionsToTreatAsEsm,omitempty"` } -type jestPreflight struct { - // Runtime is the local runtime version, for languages other than Node. - Runtime string `json:"runtime,omitempty"` - WebdriverIO *framework.WebdriverIOConfig `json:"webdriverio,omitempty"` - RunnerBinary *runnerBinary `json:"runner_binary,omitempty"` - PlaywrightProjects []playwrightProject `json:"playwright_projects,omitempty"` - VitestProjects []vitestProject `json:"vitest_projects,omitempty"` - ExecutionPath string `json:"execution_path,omitempty"` - Verdict verdict `json:"verdict"` - Command string `json:"command"` - Node string `json:"node"` - Executable string `json:"executable,omitempty"` - Prefix string `json:"prefix,omitempty"` - Arguments []string `json:"arguments,omitempty"` - Version string `json:"version"` - Projects []jestProject `json:"projects"` - Recommendation *tracerRecommendation `json:"recommendation,omitempty"` - RecommendationNotes []string `json:"recommendation_notes,omitempty"` -} - func (t *Testdrive) checkJestPreflight(ctx context.Context, output io.Writer, result *validationResult) (preflightErr error) { command, baseArgs := t.command, t.args if slices.Contains([]string{"npm", "npx", "pnpm", "yarn", "bun"}, filepath.Base(command)) { @@ -262,85 +241,6 @@ func releaseParts(version string) (int, int, bool) { // frameworkPreflight returns the selected framework's own prerequisite checks, // bound to t. A framework without them gets nil, never another framework's. -func (t *Testdrive) frameworkPreflight() func(context.Context, io.Writer, *validationResult) error { - if adapter, ok := pairedFrameworkNamed(t.framework.Name()); ok && adapter.preflight != nil { - return adapter.preflight(t) - } - return nil -} - -func (t *Testdrive) runConfigurationChecks(ctx context.Context, output io.Writer) (runErr error) { - result := validationResult{Framework: t.framework.Name(), Tracer: t.tracerLabel, CheckOnly: true, - Compatibility: verdict{Status: onboard.StatusNotExercised, Reason: "Configuration checks only; run testdrive without --check-only for paired execution."}, - Features: []featureResult{{Name: "all", Status: onboard.StatusNotExercised, Reason: "Configuration checks only."}}} - defer func() { runErr = errors.Join(runErr, finishValidation(output, t.repositoryRoot, result)) }() - if t.discoveryError != nil { - result.Error = t.discoveryError.Error() - return t.discoveryError - } - scope, err := onboard.DiscoverValidationScope(t.repositoryRoot, t.framework.Name()) - if err != nil { - result.Error = err.Error() - return err - } - result.Scope = &scope - check := checkConfigurationCIRuntimes(ctx, t.repositoryRoot, t.framework.Name()) - result.CIRuntime = &check - t.defaultToCITracer(check) - result.Tracer = t.tracerLabel - preflight := t.frameworkPreflight() - if preflight == nil { - err := fmt.Errorf("configuration checks are not supported for %s", t.framework.Name()) - result.Error = err.Error() - return err - } - if t.lang().isolatesConfigurations() && !t.isolated { - // Loading the project's dependencies can run project code even when - // nothing is installed; focused checks get the same copy as --all. - runner := *t - runner.preflight = preflight - item := configurationResult{TestCommand: onboard.TestCommand{Directory: ".", Environment: t.configurationEnvironment}} - checked, err := runner.runPreparedConfiguration(ctx, output, &item) - result.Preflight, result.CIRuntime, result.Tracer = checked.Preflight, checked.CIRuntime, checked.Tracer - if err != nil { - result.Error = err.Error() - } - return err - } - if err := preflight(ctx, output, &result); err != nil { - result.Error = err.Error() - return err - } - return nil -} - -// Full package/checkout scope coordination belongs to the configuration PR. -// Until then, nested invocation must not certify CI it did not inspect. -func checkConfigurationCIRuntimes(ctx context.Context, packageRoot, frameworkName string) onboard.RuntimeCheck { - check := frameworkCIRuntimes(ctx, packageRoot, frameworkName) - checkout := git.GetSourceRoot() - if checkout == "" { - return check - } - canonicalPackage, _ := filepath.EvalSymlinks(packageRoot) - canonicalCheckout, _ := filepath.EvalSymlinks(checkout) - if canonicalPackage == canonicalCheckout { - return check - } - // The selected provider's checkout-level configuration, not another's. - files, err := onboard.CIProviderConfigurationFiles(checkout) - if err != nil || len(files) > 0 { - location := onboard.CIProviderDisplayName() - if len(files) > 0 { - location = filepath.ToSlash(filepath.Dir(files[0])) - } - check.Status = onboard.StatusInconclusive - check.Reason = "Checkout-level " + onboard.CIProviderDisplayName() + " workflows were not validated from this package directory. Validate the checkout's CI/package scope before claiming configuration checks passed." - check.Review = append(check.Review, onboard.RuntimeFinding{Workflow: location, Status: onboard.StatusNotExercised, Code: onboard.CodeNestedCIScope, Reason: check.Reason}) - } - return check -} - func (t *Testdrive) verifyInstalledSelection(ctx context.Context, preload string, result *validationResult) error { if result.Selection == nil { return fmt.Errorf("tracer selection metadata is missing") diff --git a/internal/testdrive/language.go b/internal/testdrive/language.go index 9d42edc0c..c529a0ddf 100644 --- a/internal/testdrive/language.go +++ b/internal/testdrive/language.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "fmt" - "io" "strings" "time" @@ -12,7 +11,6 @@ import ( "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/testdrive/intake" - "github.com/DataDog/ddtest/internal/testdrive/workspace" ) // validationLanguage is everything testdrive needs from one language. Shared @@ -186,39 +184,6 @@ type environmentRequest struct { } // pairedFramework is one framework's part of paired validation. -type pairedFramework struct { - // run executes one baseline, reporting-only or probe variant. - run func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, preload string, spec runSpec) (validationRun, error) - // features runs the feature probes after a compatible pair. - features func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, preload string, baseline validationRun, result *validationResult) error - // preflight checks the framework's own prerequisites; nil has none. - preflight func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error - // telemetry matches native results to reported telemetry; nil uses - // defaultTelemetry. - telemetry func(validationRun) verdict - // probeTelemetry matches a probe control's native result to its - // telemetry; nil uses the shared file-based check. - probeTelemetry func(validationRun) verdict - // preflightAfterInstall runs preflight again once the tracer is - // installed, to inspect the dependencies that load it. - preflightAfterInstall bool - // verifyInstallation confirms that the selected runtime loads the selected tracer. - verifyInstallation func(t *Testdrive, ctx context.Context, preload string) error - // missingCoverage is the blocking check when real-suite coverage is - // missing; empty uses the default. - missingCoverage string - // outcomeTotals adds the reporting-only native outcomes summed across - // configurations to the handoff. - outcomeTotals bool -} - -// Neutral names for the paired-validation building blocks every language -// shares; the originals are named after the framework that introduced them. -type ( - nativeTest = jestTest - preflightCheck = jestPreflight -) - // defaultTelemetry matches native results to telemetry by file, name and status. func defaultTelemetry(run validationRun) verdict { return verifyJestTelemetry(run) } @@ -256,16 +221,6 @@ func languageForFramework(framework string) validationLanguage { return nil } -// pairedFrameworkNamed returns the paired-validation adapter of framework. -func pairedFrameworkNamed(framework string) (pairedFramework, bool) { - language := languageForFramework(framework) - if language == nil { - return pairedFramework{}, false - } - adapter, ok := language.pairedFrameworks()[framework] - return adapter, ok -} - // lang is the selected language; a run without a registered one gets // unsupportedLanguage, which adds nothing. func (t *Testdrive) lang() validationLanguage { diff --git a/internal/testdrive/native_results.go b/internal/testdrive/native_results.go new file mode 100644 index 000000000..a3f27481f --- /dev/null +++ b/internal/testdrive/native_results.go @@ -0,0 +1,327 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "encoding/json" + "fmt" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/testdrive/intake" +) + +// jestTest is one framework-normalized test result used by paired validation. +type jestTest struct { + WebdriverIOCapability string `json:"webdriverio_capability,omitempty"` + WebdriverIOTestID string `json:"webdriverio_test_id,omitempty"` + WebdriverIOBrowser string `json:"webdriverio_browser,omitempty"` + CucumberAttempts []string `json:"-"` + WebdriverIOAttempts []string `json:"-"` + CucumberLine int `json:"-"` + CypressAttempts []string `json:"-"` + Playwright *playwrightIdentity `json:"playwright,omitempty"` + // NativeID is the runner's own unique example identity, reported in the + // tracer's parameters metadata as scoped_id. + NativeID string `json:"native_id,omitempty"` + // telemetryClass qualifies telemetryName with its test class, which the + // tracer reports as the suite. + telemetryClass string + // probe marks the controlled probe when its native name is not probeName. + probe bool + telemetryName *string + File string `json:"file"` + Name string `json:"name"` + Status string `json:"status"` + Failure string `json:"failure,omitempty"` +} + +type nativeTest = jestTest + +var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + +// Object addresses differ between processes. Only addresses inside a +// recognizable object representation are replaced: Ruby's # +// (and #), Python's +// and Java's com.acme.Widget@1b6d3586, an array's [Ljava.lang.String;@1b6d3586 +// or [I@1b6d3586, or a lambda's Test$$Lambda$1352/0x000000980176c488@4f3b5c60, +// whose number and hidden class address differ too. Bare hexadecimal values +// are data and stay, so differing values still differ. +var ( + inspectedObjectAddress = regexp.MustCompile(`(#<[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*:)0x[0-9a-fA-F]{6,}`) + reprObjectAddress = regexp.MustCompile(`(<[A-Za-z_][A-Za-z0-9_.]* object at )0x[0-9a-fA-F]{6,}(>)`) + lambdaObject = regexp.MustCompile(`\$\$Lambda(?:\$[0-9]+)?(?:/0x[0-9a-f]+)?@[0-9a-f]{1,8}\b`) + objectHash = regexp.MustCompile(`(\b[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)+|\[+L[A-Za-z_$][A-Za-z0-9_$.]*;|\[+[ZBCSIJFD])@[0-9a-f]{1,8}\b`) +) + +// withoutObjectAddresses replaces object addresses in a failure message, +// so the same failure compares equal across runs. +func withoutObjectAddresses(message string) string { + message = inspectedObjectAddress.ReplaceAllString(message, "${1}0x…") + message = reprObjectAddress.ReplaceAllString(message, "${1}0x…${2}") + message = lambdaObject.ReplaceAllString(message, "$$$$Lambda@…") + return objectHash.ReplaceAllString(message, "$1@…") +} + +func failureSignature(message string) string { + var lines []string + for _, line := range strings.Split(withoutObjectAddresses(ansiEscape.ReplaceAllString(message, "")), "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "at ") { + lines = append(lines, line) + } + } + return strings.Join(lines, "\n") +} + +func commandDiagnostic(output []byte) string { + text := []rune(failureSignature(string(output))) + const limit = diagnosticLimit + if len(text) > limit { + const prefix = "[truncated] …" + return prefix + string(text[len(text)-(limit-len([]rune(prefix))):]) + } + return string(text) +} + +func executedTests(run validationRun) int { + count := 0 + for _, test := range run.Tests { + if test.Status == "passed" || test.Status == "failed" || (run.framework == "cucumber" && (test.Status == "pending" || test.Status == "undefined")) { + count++ + } + } + return count +} + +func outcomeKeys(run validationRun) []string { + keys := []string{fmt.Sprintf("exit=%d", run.ExitCode), "result=" + run.ResultError} + keys = append(keys, run.SuiteErrors...) + keys = append(keys, run.SuiteFailures...) + for _, test := range run.Tests { + scoped := "" + if test.NativeID != "" { + scoped = " [scoped_id: " + test.NativeID + "]" + } + keys = append(keys, repositoryTestFile(run, test.File)+" › "+test.Name+scoped+cucumberLocation(test)+playwrightLabel(test)+webdriverIOOutcome(test)+": "+test.Status+"\n"+test.Failure+playwrightOutcome(test)+cypressOutcome(test)+cucumberOutcome(test)+webdriverIOAttemptOutcome(test)) + } + for i := range keys { + keys[i] = normalizeOwnedTemporaryPath(run, keys[i]) + } + slices.Sort(keys) + return keys +} + +// compareRuns compares an uninstrumented and a reporting-only run of the same +// suite, then matches the reporting run's telemetry with telemetry. +func compareRuns(baseline, instrumented validationRun, telemetry func(validationRun) verdict) verdict { + if reason := pairTimeout(baseline, instrumented); reason != "" { + return verdict{Status: "inconclusive", Code: "command_timeout", Reason: reason} + } + if baseline.ResultError != "" { + return verdict{Status: "inconclusive", Reason: "The uninstrumented run did not produce usable results: " + reportText(baseline.ResultError) + "."} + } + if executedTests(baseline) == 0 { + return verdict{Status: "inconclusive", Reason: "The uninstrumented run did not produce results for any executed tests; check project setup using the recorded baseline command."} + } + if baseline.Facts.TestEventCount > 0 { + return verdict{Status: "inconclusive", Reason: "The baseline emitted test telemetry despite instrumentation being disabled; a clean comparison was not established."} + } + left, right := outcomeKeys(baseline), outcomeKeys(instrumented) + if !slices.Equal(left, right) { + differences := []string{} + counts := make(map[string]int) + for _, key := range left { + counts[key]++ + } + for _, key := range right { + counts[key]-- + } + for key, count := range counts { + if count == 0 { + continue + } + label, _, _ := strings.Cut(key, "\n") + side := "Without instrumentation only" + if count < 0 { + side = "With instrumentation only" + } + differences = append(differences, side+": "+label) + } + slices.Sort(differences) + return verdict{Status: "suspected regression", Reason: "Uninstrumented and reporting-only results differ.", Differences: differences} + } + if telemetry := telemetry(instrumented); telemetry.Status != "" { + return telemetry + } + return verdict{Status: "compatible", Reason: "No observed behavioral regression: test identities, outcomes, failure details, and exit code match. Existing test failures are acceptable; reporting telemetry was verified."} +} + +func verifyNativeTelemetry(run validationRun, includeSkipped bool) verdict { + if run.framework == "webdriverio" { + return verifyWebdriverIOTelemetry(run) + } + if run.framework == "cucumber" { + return verifyCucumberTelemetry(run) + } + if run.framework == "playwright" { + return verifyPlaywrightTelemetry(run) + } + if run.framework == "cypress" { + return verifyCypressTelemetry(run) + } + if run.Facts.TestEventCount == 0 { + return verdict{Status: "inconclusive", Reason: "The instrumented run sent no test events; matching test outcomes do not prove instrumentation worked."} + } + if len(run.Facts.ConfigurationErrors) > 0 { + return verdict{Status: "inconclusive", Reason: "The tracer reported configuration errors: " + strings.Join(run.Facts.ConfigurationErrors, ", ")} + } + type identity struct{ file, name, status string } + expected := make(map[identity]int) + scopedExamples := make(map[string]bool) + classes := make(map[string][]string) + for _, test := range run.Tests { + status := map[string]string{"passed": "pass", "failed": "fail"}[test.Status] + if includeSkipped && test.Status == "skipped" { + status = "skip" + } + if status != "" { + name := test.Name + if test.telemetryName != nil { + name = *test.telemetryName + } + if test.telemetryClass != "" { + key := test.File + "\x00" + name + if !slices.Contains(classes[key], test.telemetryClass) { + classes[key] = append(classes[key], test.telemetryClass) + } + name = test.telemetryClass + "\x00" + name + } + if test.NativeID != "" { + name = "scoped_id:" + test.NativeID + scopedExamples[test.File+"\x00"+name] = true + } + expected[identity{test.File, name, status}]++ + } + } + observed := make(map[identity]int) + var differences []string + for _, test := range run.Facts.Tests { + // The tracer reports source files relative to the Git checkout, and + // suite names relative to the test process directory. + file, base := test.SourceFile, run.checkoutRoot + if file == "" { + file, base = test.Suite, run.root + } + if base == "" { + base = run.root + } + if !filepath.IsAbs(file) { + file = filepath.Join(base, file) + } + file = filepath.Clean(file) + name := test.Name + if candidates := classes[file+"\x00"+name]; len(candidates) > 0 { + var matches []string + for _, class := range candidates { + if test.Suite == class || strings.HasPrefix(test.Suite, class+" at ") { + matches = append(matches, class) + } + } + // Missing or ambiguous class evidence must not match by method alone. + if len(matches) == 1 { + name = matches[0] + "\x00" + name + } + } + if len(scopedExamples) > 0 { + var parameters struct { + Metadata struct { + ScopedID string `json:"scoped_id"` + } + } + if json.Unmarshal([]byte(test.Parameters), ¶meters) == nil { + id := "scoped_id:" + parameters.Metadata.ScopedID + if scopedExamples[file+"\x00"+id] { + name = id + } + } + } + observe := func(attempt intake.TestRun) { + if attempt.Status == "pass" || attempt.Status == "fail" || includeSkipped && attempt.Status == "skip" { + observed[identity{file, name, attempt.Status}]++ + } + } + // A span sent twice is one execution; counting it again could hide a + // missing test that shares its name, such as another outline example. + spans := map[uint64]int{} + attempts := make([]intake.TestRun, 0, len(test.Attempts)) + for _, attempt := range test.Attempts { + if attempt.SpanID != 0 { + spans[attempt.SpanID]++ + if spans[attempt.SpanID] > 1 { + continue + } + } + attempts = append(attempts, attempt) + } + for _, count := range spans { + if count > 1 { + differences = append(differences, fmt.Sprintf("Duplicate telemetry: file %q, test %q, one span reported %d times", file, test.Name, count)) + } + } + var final *intake.TestRun + for i, attempt := range attempts { + if attempt.Retry && attempt.RetryReason != "external" && attempt.RetryReason != "" { + return verdict{Status: "inconclusive", Code: "unexpected_retry", Reason: "Unexpected tracer-controlled retry telemetry in the reporting-only run."} + } + if !attempt.Retry && final != nil { + observe(*final) + } + final = &attempts[i] + } + if final != nil { + observe(*final) + } + } + for key, count := range expected { + if count > observed[key] { + differences = append(differences, fmt.Sprintf("Missing telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-observed[key])) + } + } + for key, count := range observed { + if count > expected[key] { + differences = append(differences, fmt.Sprintf("Unexpected telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-expected[key])) + } + } + if len(differences) > 0 { + slices.Sort(differences) + return verdict{Status: "inconclusive", Reason: "Reported telemetry does not match the executed tests.", Differences: differences} + } + return verdict{} +} + +// Only the temporary directory created for this run is interchangeable. Keep +// file suffixes and arbitrary customer paths/names intact, including failures. +// repositoryTestFile shows a native test file relative to the checkout the run +// used, so differences name repository paths, not a temporary copy. +func repositoryTestFile(run validationRun, file string) string { + if run.root == "" || !filepath.IsAbs(file) { + return file + } + if relative, err := filepath.Rel(run.root, file); err == nil && filepath.IsLocal(relative) { + return filepath.ToSlash(relative) + } + return file +} + +func normalizeOwnedTemporaryPath(run validationRun, value string) string { + if run.temporaryRoot == "" { + return value + } + return strings.ReplaceAll(value, run.temporaryRoot+string(filepath.Separator), "/") +} diff --git a/internal/testdrive/paired_validation.go b/internal/testdrive/paired_validation.go new file mode 100644 index 000000000..c7c45ecd2 --- /dev/null +++ b/internal/testdrive/paired_validation.go @@ -0,0 +1,152 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "fmt" + "io" + + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/DataDog/ddtest/internal/testdrive/workspace" +) + +type pairedFramework struct { + // run executes one baseline, reporting-only or probe variant. + run func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, preload string, spec runSpec) (validationRun, error) + // features runs the feature probes after a compatible pair. + features func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, preload string, baseline validationRun, result *validationResult) error + // preflight checks the framework's own prerequisites; nil has none. + preflight func(t *Testdrive, ctx context.Context, output io.Writer, result *validationResult) error + // telemetry matches native results to reported telemetry; nil uses + // defaultTelemetry. + telemetry func(validationRun) verdict + // probeTelemetry matches a probe control's native result to its + // telemetry; nil uses the shared file-based check. + probeTelemetry func(validationRun) verdict + // preflightAfterInstall runs preflight again once the tracer is + // installed, to inspect the dependencies that load it. + preflightAfterInstall bool + // verifyInstallation confirms that the selected runtime loads the selected tracer. + verifyInstallation func(t *Testdrive, ctx context.Context, preload string) error + // missingCoverage is the blocking check when real-suite coverage is + // missing; empty uses the default. + missingCoverage string + // outcomeTotals adds the reporting-only native outcomes summed across + // configurations to the handoff. + outcomeTotals bool +} + +// pairedFrameworkNamed returns the paired-validation adapter of framework. +func pairedFrameworkNamed(framework string) (pairedFramework, bool) { + language := languageForFramework(framework) + if language == nil { + return pairedFramework{}, false + } + adapter, ok := language.pairedFrameworks()[framework] + return adapter, ok +} + +func (t *Testdrive) frameworkPreflight() func(context.Context, io.Writer, *validationResult) error { + if adapter, ok := pairedFrameworkNamed(t.framework.Name()); ok && adapter.preflight != nil { + return func(ctx context.Context, output io.Writer, result *validationResult) error { + return adapter.preflight(t, ctx, output, result) + } + } + return nil +} + +func (t *Testdrive) runFrameworkValidation(ctx context.Context, output io.Writer, session *workspace.Session, preload string, result *validationResult) error { + adapter, ok := pairedFrameworkNamed(t.framework.Name()) + if !ok { + return fmt.Errorf("paired validation is not supported for %s", t.framework.Name()) + } + run := func(ctx context.Context, output io.Writer, session *workspace.Session, preload string, spec runSpec) (validationRun, error) { + value, err := adapter.run(t, ctx, output, session, preload, spec) + return markTimeout(value), err + } + baseline, err := run(ctx, output, session, preload, runSpec{Name: "baseline", Role: roleBaseline, Instrumented: false}) + result.Runs = append(result.Runs, baseline.summary()) + if err != nil { + return err + } + instrumented, err := run(ctx, output, session, preload, runSpec{Name: "reporting-only", Role: roleReporting, Instrumented: true}) + result.Runs = append(result.Runs, instrumented.summary()) + if err != nil { + return err + } + if err := t.writeHTMLReport(output, result, instrumented); err != nil { + return err + } + telemetry := adapter.telemetry + if telemetry == nil { + telemetry = verifyJestTelemetry + } + result.Compatibility = compareRuns(baseline, instrumented, telemetry) + if result.Compatibility.Status == "suspected regression" { + baselineAgain, err := run(ctx, output, session, preload, runSpec{Name: "baseline-repeat", Role: roleRepeat, Instrumented: false}) + result.Runs = append(result.Runs, baselineAgain.summary()) + if err != nil { + return err + } + instrumentedAgain, err := run(ctx, output, session, preload, runSpec{Name: "reporting-only-repeat", Role: roleRepeat, Instrumented: true}) + result.Runs = append(result.Runs, instrumentedAgain.summary()) + if err != nil { + return err + } + if err := t.writeHTMLReport(output, result, instrumentedAgain); err != nil { + return err + } + result.Compatibility = compareRepeatedRuns(baseline, instrumented, baselineAgain, instrumentedAgain, telemetry) + } + // Probes would only repeat the hang; name the timeout once instead. + if reason := pairTimeout(baseline, instrumented); reason != "" { + unavailableFeatures(result, reason) + return nil + } + if compatibility, feature, ok := missingNativeResults(baseline); ok { + result.Features = []featureResult{{Name: "all", Status: "not exercised", Reason: feature}} + result.Compatibility = compatibility + return nil + } + return adapter.features(t, ctx, output, session, preload, baseline, result) +} + +// pairTimeout explains a paired comparison that testdrive's --timeout cut short. +func pairTimeout(baseline, instrumented validationRun) string { + if baseline.Timeout != "" { + return baseline.Timeout + "." + } + if instrumented.Timeout != "" { + return instrumented.Timeout + ", while the uninstrumented run finished." + } + return "" +} + +// missingNativeResults explains a baseline without test results. A failed +// command is a startup/setup problem; a successful one that ddtest can't read +// is a result-reading problem and must not send agents to change the suite. +func missingNativeResults(baseline validationRun) (verdict, string, bool) { + if len(baseline.Tests) > 0 || (baseline.ResultError == "" && baseline.ExitCode == 0) { + return verdict{}, "", false + } + if baseline.ExitCode == 0 { + return verdict{Status: "inconclusive", Code: "native_results_unreadable", Reason: reportText(baseline.ResultError) + ". The native suite exited 0, but ddtest could not read its results, so features were not validated. If this command runs the tests, this is a ddtest result-reading problem, not a suite failure: report it with the ddtest version instead of changing the repository."}, "ddtest could not read the native results of a successful suite run, so features were not validated.", true + } + return verdict{Status: "inconclusive", Code: "native_startup_failed", Reason: "Native execution failed before any test results. Resolve the baseline startup/setup diagnostic first: " + reportText(baseline.Diagnostic+" "+baseline.ResultError)}, "The native suite did not produce test results. Resolve its startup/setup diagnostic before feature or coverage validation.", true +} + +func configureScenarioEnvironment(env map[string]string, feature string) { + // Coverage requires ITR negotiation; tests_skipping is separately disabled by + // the reporting-only settings response. No skippable tests are returned there. + env["DD_CIVISIBILITY_ITR_ENABLED"] = "true" + env["DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"] = fmt.Sprint(feature == "early-flake-detection") + env["DD_TEST_EARLY_FLAKE_DETECTION_RETRY_COUNT"] = fmt.Sprint(intake.ScenarioRetryCount) + env["DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"] = fmt.Sprint(feature == "auto-retries") + env["DD_CIVISIBILITY_FLAKY_RETRY_COUNT"] = fmt.Sprint(intake.ScenarioRetryCount) + env["DD_TEST_MANAGEMENT_ENABLED"] = fmt.Sprint(feature == "quarantine" || feature == "disabled" || feature == "attempt-to-fix") + env["DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"] = fmt.Sprint(intake.ScenarioRetryCount) +} diff --git a/internal/testdrive/preflight.go b/internal/testdrive/preflight.go new file mode 100644 index 000000000..b810fb429 --- /dev/null +++ b/internal/testdrive/preflight.go @@ -0,0 +1,113 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/git" + "github.com/DataDog/ddtest/internal/onboard" +) + +// jestPreflight records configuration and runtime evidence for any framework. +type jestPreflight struct { + // Runtime is the local runtime version, for languages other than Node. + Runtime string `json:"runtime,omitempty"` + WebdriverIO *framework.WebdriverIOConfig `json:"webdriverio,omitempty"` + RunnerBinary *runnerBinary `json:"runner_binary,omitempty"` + PlaywrightProjects []playwrightProject `json:"playwright_projects,omitempty"` + VitestProjects []vitestProject `json:"vitest_projects,omitempty"` + ExecutionPath string `json:"execution_path,omitempty"` + Verdict verdict `json:"verdict"` + Command string `json:"command"` + Node string `json:"node"` + Executable string `json:"executable,omitempty"` + Prefix string `json:"prefix,omitempty"` + Arguments []string `json:"arguments,omitempty"` + Version string `json:"version"` + Projects []jestProject `json:"projects"` + Recommendation *tracerRecommendation `json:"recommendation,omitempty"` + RecommendationNotes []string `json:"recommendation_notes,omitempty"` +} + +type preflightCheck = jestPreflight + +func (t *Testdrive) runConfigurationChecks(ctx context.Context, output io.Writer) (runErr error) { + result := validationResult{Framework: t.framework.Name(), Tracer: t.tracerLabel, CheckOnly: true, + Compatibility: verdict{Status: onboard.StatusNotExercised, Reason: "Configuration checks only; run testdrive without --check-only for paired execution."}, + Features: []featureResult{{Name: "all", Status: onboard.StatusNotExercised, Reason: "Configuration checks only."}}} + defer func() { runErr = errors.Join(runErr, finishValidation(output, t.repositoryRoot, result)) }() + if t.discoveryError != nil { + result.Error = t.discoveryError.Error() + return t.discoveryError + } + scope, err := onboard.DiscoverValidationScope(t.repositoryRoot, t.framework.Name()) + if err != nil { + result.Error = err.Error() + return err + } + result.Scope = &scope + check := checkConfigurationCIRuntimes(ctx, t.repositoryRoot, t.framework.Name()) + result.CIRuntime = &check + t.defaultToCITracer(check) + result.Tracer = t.tracerLabel + preflight := t.frameworkPreflight() + if preflight == nil { + err := fmt.Errorf("configuration checks are not supported for %s", t.framework.Name()) + result.Error = err.Error() + return err + } + if t.lang().isolatesConfigurations() && !t.isolated { + // Loading the project's dependencies can run project code even when + // nothing is installed; focused checks get the same copy as --all. + runner := *t + runner.preflight = preflight + item := configurationResult{TestCommand: onboard.TestCommand{Directory: ".", Environment: t.configurationEnvironment}} + checked, err := runner.runPreparedConfiguration(ctx, output, &item) + result.Preflight, result.CIRuntime, result.Tracer = checked.Preflight, checked.CIRuntime, checked.Tracer + if err != nil { + result.Error = err.Error() + } + return err + } + if err := preflight(ctx, output, &result); err != nil { + result.Error = err.Error() + return err + } + return nil +} + +// Full package/checkout scope coordination belongs to the configuration PR. +// Until then, nested invocation must not certify CI it did not inspect. +func checkConfigurationCIRuntimes(ctx context.Context, packageRoot, frameworkName string) onboard.RuntimeCheck { + check := frameworkCIRuntimes(ctx, packageRoot, frameworkName) + checkout := git.GetSourceRoot() + if checkout == "" { + return check + } + canonicalPackage, _ := filepath.EvalSymlinks(packageRoot) + canonicalCheckout, _ := filepath.EvalSymlinks(checkout) + if canonicalPackage == canonicalCheckout { + return check + } + // The selected provider's checkout-level configuration, not another's. + files, err := onboard.CIProviderConfigurationFiles(checkout) + if err != nil || len(files) > 0 { + location := onboard.CIProviderDisplayName() + if len(files) > 0 { + location = filepath.ToSlash(filepath.Dir(files[0])) + } + check.Status = onboard.StatusInconclusive + check.Reason = "Checkout-level " + onboard.CIProviderDisplayName() + " workflows were not validated from this package directory. Validate the checkout's CI/package scope before claiming configuration checks passed." + check.Review = append(check.Review, onboard.RuntimeFinding{Workflow: location, Status: onboard.StatusNotExercised, Code: onboard.CodeNestedCIScope, Reason: check.Reason}) + } + return check +} diff --git a/internal/testdrive/preflight_dispatch_test.go b/internal/testdrive/preflight_dispatch_test.go index 2bb88f6ab..33f60b443 100644 --- a/internal/testdrive/preflight_dispatch_test.go +++ b/internal/testdrive/preflight_dispatch_test.go @@ -2,6 +2,9 @@ package testdrive import ( "bytes" + "context" + "errors" + "io" "os" "os/exec" "path/filepath" @@ -47,7 +50,9 @@ func TestEveryFrameworkWithPreflightUsesItsOwnChecks(t *testing.T) { require.True(t, ok, "add %s to this test", name) preflight := (&Testdrive{framework: runner}).frameworkPreflight() require.NotNil(t, preflight, name) - function := runtime.FuncForPC(reflect.ValueOf(preflight).Pointer()).Name() + adapter, ok := pairedFrameworkNamed(name) + require.True(t, ok, name) + function := runtime.FuncForPC(reflect.ValueOf(adapter.preflight).Pointer()).Name() switch name { case "pytest": require.Contains(t, function, ".checkPytestPreflight", name) @@ -62,6 +67,46 @@ func TestEveryFrameworkWithPreflightUsesItsOwnChecks(t *testing.T) { require.Nil(t, (&Testdrive{framework: &telemetryOnlyFramework{}}).frameworkPreflight()) } +type preflightDispatchLanguage struct { + validationLanguage + adapter pairedFramework +} + +func (language preflightDispatchLanguage) pairedFrameworks() map[string]pairedFramework { + return map[string]pairedFramework{"jest": language.adapter} +} + +func TestFrameworkPreflightBindsTheSelectedDrive(t *testing.T) { + drive := &Testdrive{framework: framework.NewJest(platform.NewJavaScript())} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + output := &bytes.Buffer{} + result := &validationResult{} + wantErr := errors.New("preflight blocked") + called := false + previous := validationLanguages + t.Cleanup(func() { validationLanguages = previous }) + validationLanguages = []validationLanguage{preflightDispatchLanguage{ + validationLanguage: javascriptLanguage{}, + adapter: pairedFramework{preflight: func(got *Testdrive, gotCtx context.Context, gotOutput io.Writer, gotResult *validationResult) error { + called = true + require.Same(t, drive, got) + require.Same(t, ctx, gotCtx) + require.ErrorIs(t, gotCtx.Err(), context.Canceled) + require.Same(t, output, gotOutput) + require.Same(t, result, gotResult) + gotResult.Tracer = "selected tracer" + return wantErr + }}, + }} + check := drive.frameworkPreflight() + require.NotNil(t, check) + require.False(t, called, "binding must not execute project configuration") + require.ErrorIs(t, check(ctx, output, result), wantErr) + require.True(t, called) + require.Equal(t, "selected tracer", result.Tracer) +} + func TestPlainCheckOnlyRunsTheSelectedFrameworkPreflight(t *testing.T) { if _, err := exec.LookPath("node"); err != nil { t.Skip("Node is required") diff --git a/internal/testdrive/python_language.go b/internal/testdrive/python_language.go index 24a5e9ba5..f9a3f1d1d 100644 --- a/internal/testdrive/python_language.go +++ b/internal/testdrive/python_language.go @@ -3,7 +3,6 @@ package testdrive import ( "context" "fmt" - "io" "maps" "os" "path/filepath" @@ -150,11 +149,9 @@ func (pythonLanguage) noResultsFinding(runtime reportRuntime, noEvents, commandF func (pythonLanguage) pairedFrameworks() map[string]pairedFramework { return map[string]pairedFramework{"pytest": { - run: (*Testdrive).runPytest, - features: (*Testdrive).runPytestFeatures, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkPytestPreflight - }, + run: (*Testdrive).runPytest, + features: (*Testdrive).runPytestFeatures, + preflight: (*Testdrive).checkPytestPreflight, telemetry: verifyPytestTelemetry, probeTelemetry: verifyPytestTelemetry, verifyInstallation: func(t *Testdrive, ctx context.Context, path string) error { diff --git a/internal/testdrive/ruby_language.go b/internal/testdrive/ruby_language.go index d0c4529a1..b79ed5150 100644 --- a/internal/testdrive/ruby_language.go +++ b/internal/testdrive/ruby_language.go @@ -159,9 +159,7 @@ func (rubyLanguage) pairedFrameworks() map[string]pairedFramework { features: func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, _ string, baseline validationRun, result *validationResult) error { return t.runRubyFeatures(ctx, output, session, baseline, result) }, - preflight: func(t *Testdrive) func(context.Context, io.Writer, *validationResult) error { - return t.checkRubyPreflight - }, + preflight: (*Testdrive).checkRubyPreflight, telemetry: verifyRubyTelemetry, preflightAfterInstall: true, missingCoverage: "Ruby application coverage was not observed. Inspect the selected Gemfile/runtime and native suite output; setup failures before tests start cannot validate compatibility or features. Check datadog-ci coverage settings after the native suite runs.", From eb92a56dd0c0fd2eaaff4d5f29ce1bdb95afb656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 6 Nov 2026 09:33:25 +0100 Subject: [PATCH 5/5] Centralize JavaScript validation command selection --- internal/onboard/README.md | 1 + internal/testdrive/javascript_command.go | 92 +++++++++++++ internal/testdrive/javascript_command_test.go | 82 ++++++++++++ internal/testdrive/javascript_language.go | 3 +- internal/testdrive/testdrive.go | 122 ++---------------- 5 files changed, 185 insertions(+), 115 deletions(-) create mode 100644 internal/testdrive/javascript_command.go create mode 100644 internal/testdrive/javascript_command_test.go diff --git a/internal/onboard/README.md b/internal/onboard/README.md index c56241b2d..7e84a7cc0 100644 --- a/internal/onboard/README.md +++ b/internal/onboard/README.md @@ -22,6 +22,7 @@ Start at the boundary that owns the behavior you need to change: | Literal command recognition without executing repository code | `internal/testcommand` | | Setup plans and bounded source edits | `internal/onboard/execution_plan.go`, `apply*.go` | | Validation preparation and execution | `internal/testdrive/testdrive.go` | +| JavaScript command selection, fallbacks and invocation parsing | `internal/testdrive/javascript_command.go` | | Shared paired-execution engine and framework adapters | `internal/testdrive/paired_validation.go` | | Shared preflight evidence and CI configuration checks | `internal/testdrive/preflight.go` | | Native result model, outcome comparison and telemetry matching | `internal/testdrive/native_results.go` | diff --git a/internal/testdrive/javascript_command.go b/internal/testdrive/javascript_command.go new file mode 100644 index 000000000..ce91bca68 --- /dev/null +++ b/internal/testdrive/javascript_command.go @@ -0,0 +1,92 @@ +package testdrive + +import ( + "fmt" + "slices" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/kballard/go-shellquote" +) + +type javascriptCommandRequest struct { + root, framework, override string + checkOnly bool + scope onboard.ValidationScope + scopeErr error +} + +// javascriptCommandSelection retains the invocation's plan and environment, +// which are needed to credit the matching CI configuration after selection. +type javascriptCommandSelection struct { + command string + args []string + plan onboard.TestCommand + environment map[string]string +} + +// selectJavaScriptCommand keeps the fallback until discovery supplies a valid +// invocation. Its errors are deferred until Run so startup recovery can replace +// an earlier report even when command discovery fails. +func selectJavaScriptCommand(request javascriptCommandRequest, command string, args []string) (javascriptCommandSelection, error) { + result := javascriptCommandSelection{command: command, args: args} + var selected string + var err error + switch request.framework { + case "jest": + if request.checkOnly { + var entry onboard.TestCommand + entry, err = onboard.JestConfigurationCommand(request.root, request.override) + selected, result.environment = entry.Command, entry.Environment + } else { + if request.override != "" { + result.plan, err = onboard.ForwardJestExecution(request.root, request.override) + } else { + result.plan, err = onboard.JestExecutionCommand(request.root) + } + selected = result.plan.Command + } + case "playwright": + selected, err = onboard.PlaywrightValidationCommandInScope(request.root, request.override, request.scope, request.scopeErr) + case "cypress": + result.plan, err = onboard.CypressExecutionCommandInScope(request.root, request.override, request.scope, request.scopeErr) + selected = result.plan.Command + case "cucumber": + result.plan, err = onboard.CucumberExecutionCommand(request.root, request.override) + selected = result.plan.Command + case "webdriverio": + selected, err = onboard.WebdriverIOValidationCommand(request.root, request.override) + case "mocha": + selected, err = onboard.MochaValidationCommandInScope(request.root, request.override, request.scope, request.scopeErr) + case "vitest": + result.plan, err = onboard.VitestValidationCommandInScope(request.root, request.override, request.scope, request.scopeErr) + selected = result.plan.Command + default: + return result, nil + } + // Jest can return a usable invocation with a discovery finding. Keep both; + // the other selectors return no executable invocation on an error. + if err != nil && request.framework != "jest" { + return result, err + } + if request.framework == "cypress" && result.plan.Cypress != nil { + result.command, result.args = "node", nil + return result, nil + } + if request.framework == "vitest" { + result.environment = result.plan.Environment + } + // Cypress and Vitest require an explicit resolved invocation. Other + // frameworks may intentionally fall back to their default command. + if selected == "" && request.framework != "cypress" && request.framework != "vitest" { + return result, err + } + words, splitErr := shellquote.Split(selected) + if splitErr != nil || len(words) == 0 { + return result, fmt.Errorf("invalid %s command: %s", displayName(request.framework), selected) + } + result.command, result.args = words[0], words[1:] + if request.framework == "jest" && result.command == "npx" && !slices.Contains(result.args, "--no-install") { + result.args = append([]string{"--no-install"}, result.args...) + } + return result, err +} diff --git a/internal/testdrive/javascript_command_test.go b/internal/testdrive/javascript_command_test.go new file mode 100644 index 000000000..34d3f9268 --- /dev/null +++ b/internal/testdrive/javascript_command_test.go @@ -0,0 +1,82 @@ +package testdrive + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/settings" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestPrepareJavaScriptExplicitCommands(t *testing.T) { + for _, tc := range []struct { + framework, command, executable string + args []string + }{ + {"jest", "npx jest --ci", "jest", []string{"--ci"}}, + {"vitest", "npx vitest run", "vitest", []string{"run"}}, + {"mocha", "npx mocha", "npx", []string{"mocha"}}, + {"playwright", "npx playwright test", "npx", []string{"playwright", "test"}}, + {"cypress", "npx cypress run", "npx", []string{"cypress", "run"}}, + {"cucumber", "npx cucumber-js", "npx", []string{"cucumber-js"}}, + {"webdriverio", "npx wdio run wdio.conf.cjs", "npx", []string{"wdio", "run", "wdio.conf.cjs"}}, + } { + t.Run(tc.framework, func(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + requireWriteFile(t, "package.json", `{"private":true}`) + viper.Reset() + viper.Set("framework", tc.framework) + viper.Set("command", tc.command) + settings.Init() + t.Cleanup(func() { viper.Reset(); settings.Init() }) + drive, err := Prepare(PrepareOptions{TracerVersion: "6.20.1", CheckOnly: true}) + require.NoError(t, err) + require.NoError(t, drive.discoveryError) + require.Equal(t, tc.executable, filepath.Base(drive.command)) + require.Equal(t, tc.args, drive.args) + require.NotNil(t, drive.preflight) + require.NoDirExists(t, filepath.Join(root, ".testoptimization"), "preparation must remain read-only") + require.NoDirExists(t, drive.session.Directory()) + }) + } +} + +func TestPrepareVitestCommandEnvironmentOverridesCI(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + requireWriteFile(t, "package.json", `{"private":true,"scripts":{"test":"vitest run"}}`) + require.NoError(t, os.MkdirAll(".github/workflows", 0700)) + requireWriteFile(t, ".github/workflows/test.yml", "jobs:\n tests:\n steps:\n - run: npm test\n env: {SHARED: ci, CI_ONLY: preserved}\n") + viper.Reset() + viper.Set("framework", "vitest") + viper.Set("command", "SHARED='two words' npx vitest run") + settings.Init() + t.Cleanup(func() { viper.Reset(); settings.Init() }) + drive, err := Prepare(PrepareOptions{TracerVersion: "6.20.1", CheckOnly: true}) + require.NoError(t, err) + require.NoError(t, drive.discoveryError) + require.Equal(t, "two words", drive.configurationEnvironment["SHARED"]) + require.Equal(t, "preserved", drive.configurationEnvironment["CI_ONLY"]) + require.Equal(t, map[string]string{"SHARED": "two words"}, drive.selectedPlan.Environment) + // Adding inherited CI values must not mutate the selected command's layer. + require.NotContains(t, drive.preflightEnvironment, "CI_ONLY") +} + +func TestJavaScriptSelectionPreservesFallbackOnDiscoveryFailure(t *testing.T) { + root := t.TempDir() + requireWriteFile(t, filepath.Join(root, "package.json"), `{"private":true}`) + for _, name := range (javascriptLanguage{}).frameworks() { + t.Run(name, func(t *testing.T) { + args := []string{"original", "two words"} + selected, err := selectJavaScriptCommand(javascriptCommandRequest{ + root: root, framework: name, override: "echo 'unterminated", + }, "fallback", args) + require.Error(t, err) + require.Equal(t, "fallback", selected.command) + require.Equal(t, args, selected.args) + }) + } +} diff --git a/internal/testdrive/javascript_language.go b/internal/testdrive/javascript_language.go index 4cd1979e4..be5bc9338 100644 --- a/internal/testdrive/javascript_language.go +++ b/internal/testdrive/javascript_language.go @@ -33,7 +33,8 @@ func (javascriptLanguage) installedTracerVersion(detected string) string { // storage; an unusable folder would otherwise surface later as registry failures. func (javascriptLanguage) prepareStorage() error { return checkTemporaryStorage() } -// Jest and the other runners select their commands in Prepare itself. +// JavaScript command selection also needs the shared CI scope discovered later +// in Prepare; selectJavaScriptCommand owns that phase. func (javascriptLanguage) selectCommand(commandRequest) (string, []string, bool, error) { return "", nil, false, nil } diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index dc433102c..9c717cb3f 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -238,105 +238,13 @@ func Prepare(configuration PrepareOptions) (*Testdrive, error) { if frameworkFeaturePolicies[runner.Name()].Configurations && !all { scope, scopeErr = onboard.DiscoverValidationScope(repositoryRoot, runner.Name()) } - if runner.Name() == "jest" && !all { - var selected string - if configurationOnly { - var entry onboard.TestCommand - entry, err = onboard.JestConfigurationCommand(repositoryRoot, strings.TrimSpace(settings.GetCommand())) - selected, preflightEnvironment = entry.Command, entry.Environment - } else if override := strings.TrimSpace(settings.GetCommand()); override != "" { - selectedPlan, err = onboard.ForwardJestExecution(repositoryRoot, override) - selected = selectedPlan.Command - } else { - selectedPlan, err = onboard.JestExecutionCommand(repositoryRoot) - selected = selectedPlan.Command - } - if err != nil { - discoveryError = err - } - if selected != "" { - words, err := shellquote.Split(selected) - if err != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Jest command: %s", selected) - } else { - command, args = words[0], words[1:] - if command == "npx" && !slices.Contains(args, "--no-install") { - args = append([]string{"--no-install"}, args...) - } - } - - } - } - // A failed discovery still runs startup recovery and replaces the earlier - // report after confirmation, so a stale PASSED result never stays current. - if runner.Name() == "playwright" && !all { - selected, resolveErr := onboard.PlaywrightValidationCommandInScope(repositoryRoot, strings.TrimSpace(settings.GetCommand()), scope, scopeErr) - if resolveErr != nil { - discoveryError = resolveErr - } else if selected != "" { - words, splitErr := shellquote.Split(selected) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Playwright command: %s", selected) - } else { - command, args = words[0], words[1:] - } - } - } - if runner.Name() == "cypress" && !all { - selectedPlan, err = onboard.CypressExecutionCommandInScope(repositoryRoot, strings.TrimSpace(settings.GetCommand()), scope, scopeErr) - if err != nil { - discoveryError = err - } else if selectedPlan.Cypress != nil { - command, args = "node", nil - } else { - words, splitErr := shellquote.Split(selectedPlan.Command) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Cypress command: %s", selectedPlan.Command) - } else { - command, args = words[0], words[1:] - } - } - } - if runner.Name() == "cucumber" && !all { - var resolveErr error - selectedPlan, resolveErr = onboard.CucumberExecutionCommand(repositoryRoot, strings.TrimSpace(settings.GetCommand())) - selected := selectedPlan.Command - if resolveErr != nil { - discoveryError = resolveErr - } else if selected != "" { - words, splitErr := shellquote.Split(selected) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Cucumber command: %s", selected) - } else { - command, args = words[0], words[1:] - } - } - } - if runner.Name() == "webdriverio" && !all { - selected, resolveErr := onboard.WebdriverIOValidationCommand(repositoryRoot, strings.TrimSpace(settings.GetCommand())) - if resolveErr != nil { - discoveryError = resolveErr - } else if selected != "" { - words, splitErr := shellquote.Split(selected) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid WebdriverIO command: %s", selected) - } else { - command, args = words[0], words[1:] - } - } - } - if runner.Name() == "mocha" && !all { - selected, resolveErr := onboard.MochaValidationCommandInScope(repositoryRoot, strings.TrimSpace(settings.GetCommand()), scope, scopeErr) - if resolveErr != nil { - discoveryError = resolveErr - } else if selected != "" { - words, splitErr := shellquote.Split(selected) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Mocha command: %s", selected) - } else { - command, args = words[0], words[1:] - } - } + if language == "javascript" && !all { + selected, selectionErr := selectJavaScriptCommand(javascriptCommandRequest{ + root: repositoryRoot, framework: runner.Name(), override: strings.TrimSpace(settings.GetCommand()), + checkOnly: configurationOnly, scope: scope, scopeErr: scopeErr, + }, command, args) + command, args = selected.command, selected.args + selectedPlan, preflightEnvironment, discoveryError = selected.plan, selected.environment, selectionErr } label := selectedLanguage.tracerPackage() + "@" + version @@ -344,18 +252,6 @@ func Prepare(configuration PrepareOptions) (*Testdrive, error) { if !supportsValidationFramework(runner.Name()) { session.Persist(filepath.Join(repositoryRoot, ".testoptimization", "testdrive", session.ID())) } - if runner.Name() == "vitest" && !all { - selectedPlan, discoveryError = onboard.VitestValidationCommandInScope(repositoryRoot, strings.TrimSpace(settings.GetCommand()), scope, scopeErr) - if discoveryError == nil { - words, splitErr := shellquote.Split(selectedPlan.Command) - if splitErr != nil || len(words) == 0 { - discoveryError = fmt.Errorf("invalid Vitest command: %s", selectedPlan.Command) - } else { - command, args = words[0], words[1:] - } - preflightEnvironment = selectedPlan.Environment - } - } originalCommand := shellquote.Join(append([]string{command}, args...)...) if selectedPlan.Cypress != nil { originalCommand = selectedPlan.Command @@ -391,9 +287,7 @@ func Prepare(configuration PrepareOptions) (*Testdrive, error) { if matched != nil { matched = onboard.WithCIProviderDefaults(matched) } - configurationEnvironment = mergeConfigurationEnvironment(matched, preflightEnvironment) - configurationEnvironment = mergeConfigurationEnvironment(configurationEnvironment, selectedPlan.Environment) - configurationEnvironment = mergeConfigurationEnvironment(configurationEnvironment, commandAssignments) + configurationEnvironment = mergeConfigurationEnvironment(matched, preflightEnvironment, selectedPlan.Environment, commandAssignments) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel()