From 963d455ff8ef2da91545e56c22fd029ab7c15428 Mon Sep 17 00:00:00 2001 From: simonfaltum <4305831+simonfaltum@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:52:10 +0000 Subject: [PATCH 1/5] Allow AI agents to submit feedback without confirmation --- .../cli/feedback-agent-submissions.md | 1 + acceptance/cmd/feedback/output.txt | 112 +++++++++-- acceptance/cmd/feedback/script | 16 +- cmd/feedback/feedback.go | 133 ++----------- cmd/feedback/feedback_test.go | 181 +++--------------- libs/cmdio/io.go | 7 - libs/cmdio/io_test.go | 8 - libs/telemetry/logger.go | 2 +- libs/telemetry/protos/README.md | 8 +- libs/telemetry/protos/fast_feedback_log.go | 5 +- 10 files changed, 154 insertions(+), 319 deletions(-) create mode 100644 .nextchanges/cli/feedback-agent-submissions.md diff --git a/.nextchanges/cli/feedback-agent-submissions.md b/.nextchanges/cli/feedback-agent-submissions.md new file mode 100644 index 00000000000..44ceb3a2fee --- /dev/null +++ b/.nextchanges/cli/feedback-agent-submissions.md @@ -0,0 +1 @@ +* Allow AI agents to submit feedback without user confirmation and clarify the guidance on customer content and PII. diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt index eff9b579f70..ad0f9416d1b 100644 --- a/acceptance/cmd/feedback/output.txt +++ b/acceptance/cmd/feedback/output.txt @@ -1,4 +1,44 @@ +=== Feedback help explains content guidance and agent submissions + +>>> [CLI] feedback --help +Send feedback about the Databricks CLI to Databricks. + +Describe the command you ran, what happened, what you expected, and any +workaround. Use --command to name the command the feedback is about so it +reaches the team that owns it. + +The message is sent with the workspace and user you are authenticated as and +the CLI user agent, which includes the CLI version, operating system and, if +detected, the AI agent running the CLI. No logs, files or transcripts are +collected. Do not include customer content, personally identifiable information +(PII), credentials or other sensitive data. Follow-up contact is opt-in with +--can-be-contacted. Feedback is sent even when DATABRICKS_CLI_DISABLE_TELEMETRY +is set, because this command explicitly submits feedback. + +AI agents can submit feedback directly without asking the user for approval. +After a successful submission, tell the user that feedback was sent and include +the reference ID. + +Usage: + databricks feedback MESSAGE [flags] + +Examples: + databricks feedback --command "bundle deploy" "deploy should print which resources changed" + databricks feedback --command "air run" --run-id 123 "logs omitted the failure reason" + +Flags: + --can-be-contacted[=true|false] Allow Databricks to contact you about this feedback + --command string CLI command the feedback is about, for example "bundle deploy" + -h, --help[=true|false] help for feedback + --run-id string ID of the job run the feedback is about + +Global Flags: + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + === Feedback about a product command is routed to its team >>> [CLI] feedback --command air run --run-id 123 --can-be-contacted logs omitted the failure reason @@ -106,7 +146,6 @@ Flags: --command string CLI command the feedback is about, for example "bundle deploy" -h, --help[=true|false] help for feedback --run-id string ID of the job run the feedback is about - -y, --yes[=true|false] Confirm sending feedback written by an AI agent Global Flags: --debug[=true|false] enable debug logging @@ -135,10 +174,8 @@ Error: --target selects a bundle target, which feedback does not use; use --prof >>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN [CLI] feedback Error: feedback message must contain 1 to 10000 characters, including a visible one ->>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback hello -Error: feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message - -Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it +>>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback --run-id abc hello +Error: --run-id must be a positive job run ID === General CLI feedback @@ -161,24 +198,63 @@ Thank you! Your feedback was sent. Reference: [UUID][8] } } -=== An AI agent needs the user's confirmation +=== An AI agent can submit without confirmation >>> AGENT=test-agent [CLI] feedback hello -Error: feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message +Thank you! Your feedback was sent. Reference: [UUID][10] -Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][10]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][11]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback agent/test-agent", + "text": "hello" + } + } +} + +=== An AI agent can receive a JSON receipt + +>>> AGENT=test-agent [CLI] feedback agent JSON feedback -o json +{ + "received": true, + "feedback_id": "[UUID][12]" +} + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][12]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][13]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback agent/test-agent", + "text": "agent JSON feedback" + } + } +} -=== An AI agent with --yes +=== The deprecated --yes flag remains accepted >>> AGENT=test-agent [CLI] feedback --yes hello -Thank you! Your feedback was sent. Reference: [UUID][10] +Flag --yes has been deprecated, feedback is sent without confirmation +Thank you! Your feedback was sent. Reference: [UUID][14] >>> print_requests.py //telemetry-ext { - "frontend_log_event_id": "[UUID][10]", + "frontend_log_event_id": "[UUID][14]", "entry": { "fast_feedback_log": { - "reference_event_id": "[UUID][11]", + "reference_event_id": "[UUID][15]", "parent_ui": "DeveloperEcosystem.DatabricksCLI", "origin": "cli", "can_be_contacted": false, @@ -192,14 +268,14 @@ Thank you! Your feedback was sent. Reference: [UUID][10] === Feedback is sent even with telemetry disabled >>> DATABRICKS_CLI_DISABLE_TELEMETRY=1 [CLI] feedback telemetry is disabled -Thank you! Your feedback was sent. Reference: [UUID][12] +Thank you! Your feedback was sent. Reference: [UUID][16] >>> print_requests.py //telemetry-ext { - "frontend_log_event_id": "[UUID][12]", + "frontend_log_event_id": "[UUID][16]", "entry": { "fast_feedback_log": { - "reference_event_id": "[UUID][13]", + "reference_event_id": "[UUID][17]", "parent_ui": "DeveloperEcosystem.DatabricksCLI", "origin": "cli", "can_be_contacted": false, @@ -213,14 +289,14 @@ Thank you! Your feedback was sent. Reference: [UUID][12] === A broken bundle does not block feedback >>> withdir broken-bundle [CLI] feedback bundle config does not block feedback -Thank you! Your feedback was sent. Reference: [UUID][14] +Thank you! Your feedback was sent. Reference: [UUID][18] >>> print_requests.py //telemetry-ext { - "frontend_log_event_id": "[UUID][14]", + "frontend_log_event_id": "[UUID][18]", "entry": { "fast_feedback_log": { - "reference_event_id": "[UUID][15]", + "reference_event_id": "[UUID][19]", "parent_ui": "DeveloperEcosystem.DatabricksCLI", "origin": "cli", "can_be_contacted": false, diff --git a/acceptance/cmd/feedback/script b/acceptance/cmd/feedback/script index deed5673fc3..cc2ada0d24d 100644 --- a/acceptance/cmd/feedback/script +++ b/acceptance/cmd/feedback/script @@ -5,6 +5,9 @@ print_feedback() { | .entry.fast_feedback_log.user_agent |= (split(" ") | map(select(test("^(cli|cmd|run-id|agent)/"))) | join(" "))' } +title "Feedback help explains content guidance and agent submissions" +trace $CLI feedback --help + title "Feedback about a product command is routed to its team" trace $CLI feedback --command "air run" --run-id 123 --can-be-contacted " logs omitted the failure reason " print_feedback @@ -32,16 +35,21 @@ musterr trace $CLI feedback --target dev "hello" title "Input errors are reported before authenticating" musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN $CLI feedback " " -musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent $CLI feedback "hello" +musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent $CLI feedback --run-id abc "hello" title "General CLI feedback" trace $CLI feedback "the CLI should support ..." print_feedback -title "An AI agent needs the user's confirmation" -musterr trace AGENT=test-agent $CLI feedback "hello" +title "An AI agent can submit without confirmation" +trace AGENT=test-agent $CLI feedback "hello" < /dev/null +print_feedback + +title "An AI agent can receive a JSON receipt" +trace AGENT=test-agent $CLI feedback "agent JSON feedback" -o json < /dev/null +print_feedback -title "An AI agent with --yes" +title "The deprecated --yes flag remains accepted" trace AGENT=test-agent $CLI feedback --yes "hello" print_feedback diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go index 31cbfdc3b52..6224be7f955 100644 --- a/cmd/feedback/feedback.go +++ b/cmd/feedback/feedback.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "strconv" "strings" "time" @@ -14,13 +13,10 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" ) @@ -52,14 +48,12 @@ type options struct { command string runID string canBeContacted bool - yes bool } // submission is the validated input, checked before authenticating so that a // mistake is reported without a login prompt or a network call. type submission struct { text string - command string component string origin string runID string @@ -84,14 +78,14 @@ reaches the team that owns it. The message is sent with the workspace and user you are authenticated as and the CLI user agent, which includes the CLI version, operating system and, if detected, the AI agent running the CLI. No logs, files or transcripts are -collected. Do not include credentials or sensitive data. Follow-up contact is -opt-in with --can-be-contacted. Feedback is sent even when -DATABRICKS_CLI_DISABLE_TELEMETRY is set, because you asked to send it. - -AI agents: submit feedback only with the user's approval. When an agent is -detected, the command asks for confirmation, or requires --yes when no terminal -is attached. Show the user the exact message and the flags you pass with it, -such as --can-be-contacted, and pass --yes only after the user approves it.` +collected. Do not include customer content, personally identifiable information +(PII), credentials or other sensitive data. Follow-up contact is opt-in with +--can-be-contacted. Feedback is sent even when DATABRICKS_CLI_DISABLE_TELEMETRY +is set, because this command explicitly submits feedback. + +AI agents can submit feedback directly without asking the user for approval. +After a successful submission, tell the user that feedback was sent and include +the reference ID.` cmd.Example = ` databricks feedback --command "bundle deploy" "deploy should print which resources changed" databricks feedback --command "air run" --run-id 123 "logs omitted the failure reason"` @@ -99,7 +93,8 @@ such as --can-be-contacted, and pass --yes only after the user approves it.` cmd.Flags().StringVar(&opts.command, "command", "", `CLI command the feedback is about, for example "bundle deploy"`) cmd.Flags().StringVar(&opts.runID, "run-id", "", "ID of the job run the feedback is about") cmd.Flags().BoolVar(&opts.canBeContacted, "can-be-contacted", false, "Allow Databricks to contact you about this feedback") - cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, "Confirm sending feedback written by an AI agent") + cmd.Flags().BoolP("yes", "y", false, "Deprecated; feedback is sent without confirmation") + cmd.Flags().MarkDeprecated("yes", "feedback is sent without confirmation") cmd.Args = checkArgs @@ -110,14 +105,11 @@ such as --can-be-contacted, and pass --yes only after the user approves it.` if err != nil { return err } - if err := checkApproval(cmd.Context(), opts.yes); err != nil { - return err - } cmd.SetContext(root.SkipLoadBundle(cmd.Context())) return root.MustWorkspaceClient(cmd, args) } cmd.RunE = func(cmd *cobra.Command, args []string) error { - return send(cmd.Context(), s, opts.yes) + return send(cmd.Context(), s) } } @@ -152,7 +144,6 @@ func newSubmission(cmd *cobra.Command, message string, opts *options) (submissio } s := submission{ text: text, - command: strings.Join(strings.Fields(opts.command), " "), component: component, origin: origin, canBeContacted: opts.canBeContacted, @@ -167,7 +158,7 @@ func newSubmission(cmd *cobra.Command, message string, opts *options) (submissio return s, nil } -func send(ctx context.Context, s submission, yes bool) error { +func send(ctx context.Context, s submission) error { if s.runID != "" { ctx = useragent.InContext(ctx, runIDKey, s.runID) } @@ -177,17 +168,6 @@ func send(ctx context.Context, s submission, yes bool) error { ctx = useragent.InContext(ctx, useragent.AgentKey, provider) } cfg := cmdctx.WorkspaceClient(ctx).Config - if needsApproval(yes) { - confirmed, err := confirm(ctx, s, workspaceLabel(cfg)) - if err != nil { - return err - } - if !confirmed { - // confirm told the user; the exit code tells the agent that ran the - // command that nothing was sent. - return root.ErrAlreadyPrinted - } - } id := uuid.New().String() sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) @@ -229,9 +209,8 @@ func validateMessage(message string) (string, error) { // resolveCommand returns the component and origin for --command. The first word // picks the owning team. The command is not looked up, which would mean // building the command tree; its words are only limited to command-name -// characters, which keeps flags, paths and control sequences out of origin and -// the approval prompt. The origin joins the words with underscores, as in -// bundle_deploy. +// characters, which keeps flags, paths and control sequences out of origin. +// The origin joins the words with underscores, as in bundle_deploy. func resolveCommand(command string) (string, string, error) { words := strings.Fields(command) if len(words) == 0 { @@ -263,90 +242,6 @@ func isCommandWord(word string) bool { return true } -// needsApproval reports whether the user must approve the feedback before it is -// sent: an AI agent is running the CLI and did not pass --yes. -func needsApproval(yes bool) bool { - return useragent.AgentProvider() != "" && !yes -} - -// checkApproval returns an error when the user must approve the feedback but -// there is no terminal to ask in. It needs only the terminal, so it runs before -// authenticating. confirm shows the summary on stderr and reads a plain line -// from stdin, so it works where cmdio.IsPromptSupported says no (TERM=dumb, -// MSYS2 pipes). -func checkApproval(ctx context.Context, yes bool) error { - if !needsApproval(yes) || (cmdio.IsInputTTY(ctx) && cmdio.IsErrorTTY(ctx)) { - return nil - } - return errors.New("feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message\n\n" + - agent.AgentNote("show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it")) -} - -// confirm asks the user to approve feedback an AI agent is about to send. It -// returns false without error if the user declines. -func confirm(ctx context.Context, s submission, workspace string) (bool, error) { - cmdio.LogString(ctx, "The following feedback will be sent to Databricks:\n\n"+summary(s, workspace)) - confirmed, err := cmdio.AskYesOrNo(ctx, "Send this feedback?") - switch { - case errors.Is(err, io.EOF): - // Ctrl-D ends the input without an answer or a newline. - cmdio.LogString(ctx, "\nFeedback not sent.") - return false, nil - case err != nil: - return false, err - case !confirmed: - cmdio.LogString(ctx, "Feedback not sent.") - } - return confirmed, nil -} - -// workspaceLabel names the workspace feedback is filed in. A unified host -// serves several workspaces, so there the host alone does not identify it. -func workspaceLabel(cfg *config.Config) string { - if id := cfg.WorkspaceID; id != "" && id != profile.WorkspaceIDNone { - return fmt.Sprintf("%s (workspace ID %s)", cfg.Host, id) - } - return cfg.Host -} - -// summary returns the message, the workspace it is filed in and the choices -// sent with it, for the user to approve. -func summary(s submission, workspace string) string { - var b strings.Builder - // Indent the message so that none of its lines can pass for one of the - // fields below it. - b.WriteString(" " + strings.ReplaceAll(escapeInvisible(s.text), "\n", "\n ")) - b.WriteString("\n\n") - fmt.Fprintf(&b, "Workspace: %s\n", workspace) - if s.command != "" { - fmt.Fprintf(&b, "Command: %s\n", s.command) - } - if s.runID != "" { - fmt.Fprintf(&b, "Run ID: %s\n", s.runID) - } - contact := "no" - if s.canBeContacted { - contact = "yes" - } - fmt.Fprintf(&b, "Databricks may contact you: %s\n", contact) - return b.String() -} - -// escapeInvisible escapes the runes that invisible reports, so none of them can -// hide, reorder or overwrite part of the message in the terminal. -func escapeInvisible(text string) string { - var b strings.Builder - for _, r := range text { - if invisible(r) { - quoted := strconv.QuoteRuneToASCII(r) - b.WriteString(quoted[1 : len(quoted)-1]) - continue - } - b.WriteRune(r) - } - return b.String() -} - // invisible reports whether a terminal would not show r as written, other than // a newline or tab: characters that are not graphic, such as escape sequences, // bidi overrides and zero-width characters, and the variation selectors and diff --git a/cmd/feedback/feedback_test.go b/cmd/feedback/feedback_test.go index e9ff919ff9d..b2a69b7c1dd 100644 --- a/cmd/feedback/feedback_test.go +++ b/cmd/feedback/feedback_test.go @@ -9,17 +9,14 @@ import ( "strings" "testing" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -57,8 +54,6 @@ const ( runIDError = "--run-id must be a positive job run ID" ) -const testWorkspace = "https://workspace.test" - // newTestFeedbackCommand returns the feedback command under a root with the // root's bundle target flags. func newTestFeedbackCommand() *cobra.Command { @@ -96,23 +91,6 @@ func TestCheckArgs(t *testing.T) { } } -func TestWorkspaceLabel(t *testing.T) { - tests := []struct { - name string - workspaceID string - want string - }{ - {name: "workspace host", want: testWorkspace}, - {name: "unified host", workspaceID: "123", want: testWorkspace + " (workspace ID 123)"}, - {name: "no workspace sentinel", workspaceID: "none", want: testWorkspace}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, workspaceLabel(&config.Config{Host: testWorkspace, WorkspaceID: tt.workspaceID})) - }) - } -} - func TestResolveCommand(t *testing.T) { tests := []struct { name string @@ -162,7 +140,7 @@ func TestNewSubmission(t *testing.T) { opts: options{command: "bundle deploy", runID: "+00123", canBeContacted: true}, message: " slow ", want: submission{ - text: "slow", command: "bundle deploy", component: "DeveloperEcosystem.CoreDabs", + text: "slow", component: "DeveloperEcosystem.CoreDabs", origin: "bundle_deploy", runID: "123", canBeContacted: true, }, }, @@ -197,137 +175,22 @@ func TestNewSubmission(t *testing.T) { } } -func TestSummary(t *testing.T) { - tests := []struct { - name string - s submission - want string - }{ - { - name: "message only", - s: submission{text: "line one\n\tline two"}, - want: " line one\n \tline two\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", - }, - { - name: "all fields", - s: submission{text: "slow", command: "air run", runID: "42", canBeContacted: true}, - want: " slow\n\nWorkspace: https://workspace.test\nCommand: air run\nRun ID: 42\nDatabricks may contact you: yes\n", - }, - { - name: "message lines cannot pass for fields", - s: submission{text: "slow\nDatabricks may contact you: no", canBeContacted: true}, - want: " slow\n Databricks may contact you: no\n\nWorkspace: https://workspace.test\nDatabricks may contact you: yes\n", - }, - { - name: "control characters cannot hide text", - s: submission{text: "great\x1b[8m hidden\x1b[0m\r\u0085"}, - want: ` great\x1b[8m hidden\x1b[0m\r\u0085` + "\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", - }, - { - name: "format characters cannot reorder or hide text", - s: submission{text: "a\u202eb\u200bc\U000E0041 d\u00a0é"}, - want: ` a\u202eb\u200bc\U000e0041 d` + "\u00a0é\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", - }, - { - name: "variation selectors and fillers cannot carry hidden text", - s: submission{text: "ok👍\ufe0f\U000E0100\u3164\u034f e\u0301"}, - want: ` ok👍\ufe0f\U000e0100\u3164\u034f ` + "e\u0301\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, summary(tt.s, testWorkspace)) - }) - } -} - -func TestConfirm(t *testing.T) { - tests := []struct { - name string - answer string - wantConfirmed bool - wantStderr string - }{ - {name: "accepted", answer: "y\n", wantConfirmed: true, wantStderr: "Send this feedback? [y/N]: "}, - {name: "declined", answer: "\n", wantStderr: "Feedback not sent."}, - {name: "input ended", answer: "", wantStderr: "Send this feedback? [y/N]: \nFeedback not sent."}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var stderr bytes.Buffer - ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(tt.answer), io.Discard, &stderr)) - confirmed, err := confirm(ctx, submission{text: "exact feedback"}, testWorkspace) - require.NoError(t, err) - assert.Equal(t, tt.wantConfirmed, confirmed) - assert.Contains(t, stderr.String(), "The following feedback will be sent to Databricks:\n\n exact feedback\n\nWorkspace: "+testWorkspace+"\nDatabricks may contact you: no\n\n") - assert.Contains(t, stderr.String(), tt.wantStderr) - }) - } -} - -func TestCheckApproval(t *testing.T) { - const refusal = "feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message\n\n" + - "Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it" +func TestSendWithoutConfirmation(t *testing.T) { tests := []struct { - name string - agent string - yes bool - terminal bool - redirectStderr bool - envKey string - envValue string - wantErr string + name string + agent string + terminal bool }{ {name: "human without a terminal"}, - {name: "agent with --yes", agent: "test-agent", yes: true}, + {name: "agent without a terminal", agent: "test-agent"}, {name: "agent with a terminal", agent: "test-agent", terminal: true}, - {name: "agent with a terminal and NO_COLOR", agent: "test-agent", terminal: true, envKey: "NO_COLOR", envValue: "1"}, - {name: "agent with a dumb terminal", agent: "test-agent", terminal: true, envKey: "TERM", envValue: "dumb"}, - {name: "agent with stderr redirected", agent: "test-agent", terminal: true, redirectStderr: true, wantErr: refusal}, - {name: "agent without a terminal", agent: "test-agent", wantErr: refusal}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - testutil.SetAgent(t, tt.agent) - ctx := t.Context() - if tt.envKey != "" { - ctx = env.Set(ctx, tt.envKey, tt.envValue) - } - var in io.Reader = strings.NewReader("") - if tt.terminal { - in = cmdio.FakeTTYReader(in) - } - stderr := io.Discard - if !tt.redirectStderr { - stderr = cmdio.FakeTTY(stderr) - } - ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, in, io.Discard, stderr, "", "")) - err := checkApproval(ctx, tt.yes) - if tt.wantErr != "" { - assert.EqualError(t, err, tt.wantErr) - return - } - assert.NoError(t, err) - }) - } -} - -func TestSendAsksBeforeSending(t *testing.T) { - tests := []struct { - name string - answer string - wantSent []string - wantErr error - }{ - {name: "approved", answer: "y\n", wantSent: []string{"exact feedback"}}, - {name: "declined", answer: "n\n", wantErr: root.ErrAlreadyPrinted}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Start the server and the client before SetAgent clears the // environment: on Windows, sockets need SystemRoot. server := testserver.New(t) - var sent []string + var sent []protos.FastFeedbackLog server.Handle(http.MethodPost, "/telemetry-ext", func(req testserver.Request) any { // The handler runs on the server's goroutine, so it must not // call require. @@ -336,29 +199,37 @@ func TestSendAsksBeforeSending(t *testing.T) { for _, raw := range body.ProtoLogs { var entry protos.FrontendLog if assert.NoError(t, json.Unmarshal([]byte(raw), &entry)) { - sent = append(sent, entry.Entry.FastFeedbackLog.Text) + sent = append(sent, *entry.Entry.FastFeedbackLog) } } return telemetry.ResponseBody{NumProtoSuccess: int64(len(body.ProtoLogs))} }) w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) require.NoError(t, err) - testutil.SetAgent(t, "test-agent") + testutil.SetAgent(t, tt.agent) var stdout, stderr bytes.Buffer - ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(tt.answer), &stdout, &stderr)) + var in io.Reader = strings.NewReader("") + var errOut io.Writer = &stderr + if tt.terminal { + in = cmdio.FakeTTYReader(in) + errOut = cmdio.FakeTTY(errOut) + } + ctx := t.Context() + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, in, &stdout, errOut, "", "")) ctx = cmdctx.SetWorkspaceClient(ctx, w) ctx = cmdctx.GenerateExecId(ctx) - err = send(ctx, submission{text: "exact feedback", component: defaultComponent, origin: "cli"}, false) + err = send(ctx, submission{text: "exact feedback", component: defaultComponent, origin: "cli"}) - assert.ErrorIs(t, err, tt.wantErr) - assert.Equal(t, tt.wantSent, sent) - assert.Contains(t, stderr.String(), " exact feedback\n") - if tt.wantErr == nil { - assert.Contains(t, stdout.String(), "Thank you! Your feedback was sent.") - } else { - assert.Empty(t, stdout.String()) + require.NoError(t, err) + require.Len(t, sent, 1) + assert.Equal(t, "exact feedback", sent[0].Text) + assert.False(t, sent[0].CanBeContacted) + if tt.agent != "" { + assert.Contains(t, sent[0].UserAgent, "agent/"+tt.agent) } + assert.Empty(t, stderr.String()) + assert.Contains(t, stdout.String(), "Thank you! Your feedback was sent. Reference: ") }) } } diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go index 86b3e24d49a..4c369b49ab8 100644 --- a/libs/cmdio/io.go +++ b/libs/cmdio/io.go @@ -132,13 +132,6 @@ func IsInputTTY(ctx context.Context) bool { return c.capabilities.stdinIsTTY } -// IsErrorTTY reports whether stderr is a terminal, so that the user sees the -// messages and questions written there. Like IsInputTTY, it ignores TERM=dumb. -func IsErrorTTY(ctx context.Context) bool { - c := fromContext(ctx) - return c.capabilities.stderrIsTTY -} - // IsPagerSupported reports whether stdin, stdout, and stderr are all interactive // terminals. This is the requirement for a full-screen or navigable output // program: unlike IsPromptSupported it also checks stdout, so it returns false diff --git a/libs/cmdio/io_test.go b/libs/cmdio/io_test.go index 91ce84e6c23..6c69f1923d5 100644 --- a/libs/cmdio/io_test.go +++ b/libs/cmdio/io_test.go @@ -53,14 +53,6 @@ func TestIsInputTTY(t *testing.T) { assert.False(t, IsInputTTY(ctx)) } -func TestIsErrorTTY(t *testing.T) { - ctx := InContext(t.Context(), &cmdIO{capabilities: Capabilities{stderrIsTTY: true, dumb: true}}) - assert.True(t, IsErrorTTY(ctx)) - - ctx = InContext(t.Context(), &cmdIO{capabilities: Capabilities{stdinIsTTY: true}}) - assert.False(t, IsErrorTTY(ctx)) -} - func TestHasTerminal(t *testing.T) { for _, tc := range []struct { name string diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index 3a4f6d7edf6..569bf52c95e 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -324,7 +324,7 @@ func authenticate(r *http.Request, cfg *config.Config, running *atomic.Int32) er } // Send uploads logs in a single request and returns an error unless all of them -// were accepted. It is for events the user explicitly asked to send, so unlike +// were accepted. It is for explicit submissions, so unlike // Upload it ignores DATABRICKS_CLI_DISABLE_TELEMETRY and never retries. func Send(ctx context.Context, cfg *config.Config, logs []protos.FrontendLog) error { protoLogs, err := marshalLogs(logs) diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 76c91e5efd0..7af722e84cf 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -21,10 +21,10 @@ resolves no auth (e.g. `databricks bundle schema`), no event for that line is up text line prints again, even though the agent saw the hint. `databricks feedback` sends one `fast_feedback_log` entry, the event the in-product feedback widget -records, instead of a `databricks_cli_log`. The user asked for it to be sent, so it is uploaded -immediately in a single attempt even when `DATABRICKS_CLI_DISABLE_TELEMETRY` is set. When an AI -agent is detected, the user must first confirm the message. Agent and run context ride in the -entry's `user_agent`. +records, instead of a `databricks_cli_log`. This command explicitly submits feedback, so it uploads +immediately in a single attempt even when `DATABRICKS_CLI_DISABLE_TELEMETRY` is set. AI agents can +submit directly without user confirmation. A successful submission prints a receipt. Agent and +run context ride in the entry's `user_agent`. `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED diff --git a/libs/telemetry/protos/fast_feedback_log.go b/libs/telemetry/protos/fast_feedback_log.go index 98ead0b5abe..eb0252670f0 100644 --- a/libs/telemetry/protos/fast_feedback_log.go +++ b/libs/telemetry/protos/fast_feedback_log.go @@ -22,8 +22,7 @@ type FastFeedbackLog struct { UserAgent string `json:"user_agent,omitempty"` - // Feedback text written by the user. Unlike other telemetry it is user-authored - // by design: the user asked to send it, and confirms it first when an AI agent - // wrote it. + // Feedback text supplied by a user or an AI agent. Unlike other telemetry, + // feedback deliberately includes the submitted message. Text string `json:"text,omitempty"` } From c52b183622519e0a5c6303c292f2a6584a181616 Mon Sep 17 00:00:00 2001 From: simonfaltum <4305831+simonfaltum@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:56:04 +0000 Subject: [PATCH 2/5] Link feedback changelog entry to PR --- .nextchanges/cli/feedback-agent-submissions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/feedback-agent-submissions.md b/.nextchanges/cli/feedback-agent-submissions.md index 44ceb3a2fee..68e5e548020 100644 --- a/.nextchanges/cli/feedback-agent-submissions.md +++ b/.nextchanges/cli/feedback-agent-submissions.md @@ -1 +1 @@ -* Allow AI agents to submit feedback without user confirmation and clarify the guidance on customer content and PII. +* Allow AI agents to submit feedback without user confirmation and clarify the guidance on customer content and PII. ([#7795](https://github.com/databricks/cli/pull/7795)) From e04637aef47513639a59a6a22e83f135034108cc Mon Sep 17 00:00:00 2001 From: simonfaltum <4305831+simonfaltum@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:10:16 +0000 Subject: [PATCH 3/5] Honor explicit feedback declines from AI agents --- .nextchanges/cli/feedback-agent-submissions.md | 2 +- acceptance/cmd/feedback/output.txt | 18 +++++++++++++++++- acceptance/cmd/feedback/script | 6 ++++++ cmd/feedback/feedback.go | 11 +++++++++-- cmd/feedback/feedback_test.go | 4 +++- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.nextchanges/cli/feedback-agent-submissions.md b/.nextchanges/cli/feedback-agent-submissions.md index 68e5e548020..1eaf3c39b6e 100644 --- a/.nextchanges/cli/feedback-agent-submissions.md +++ b/.nextchanges/cli/feedback-agent-submissions.md @@ -1 +1 @@ -* Allow AI agents to submit feedback without user confirmation and clarify the guidance on customer content and PII. ([#7795](https://github.com/databricks/cli/pull/7795)) +* Allow AI agents to submit feedback without confirmation and clarify the guidance on customer content and PII; deprecate `--yes` / `-y` while continuing to reject explicit false values from agents. ([#7795](https://github.com/databricks/cli/pull/7795)) diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt index ad0f9416d1b..cf50f546df8 100644 --- a/acceptance/cmd/feedback/output.txt +++ b/acceptance/cmd/feedback/output.txt @@ -177,6 +177,10 @@ Error: feedback message must contain 1 to 10000 characters, including a visible >>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback --run-id abc hello Error: --run-id must be a positive job run ID +>>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback --yes=false hello +Flag --yes has been deprecated, confirmation is no longer required; omit this flag +Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation + === General CLI feedback >>> [CLI] feedback the CLI should support ... @@ -243,10 +247,22 @@ Thank you! Your feedback was sent. Reference: [UUID][10] } } +=== An explicit rejection from an AI agent sends nothing + +>>> AGENT=test-agent [CLI] feedback --yes=false hello +Flag --yes has been deprecated, confirmation is no longer required; omit this flag +Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation + +>>> AGENT=test-agent [CLI] feedback -y=false hello +Flag --yes has been deprecated, confirmation is no longer required; omit this flag +Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation + +>>> find.py ^out\.requests\.txt$ --expect 0 + === The deprecated --yes flag remains accepted >>> AGENT=test-agent [CLI] feedback --yes hello -Flag --yes has been deprecated, feedback is sent without confirmation +Flag --yes has been deprecated, confirmation is no longer required; omit this flag Thank you! Your feedback was sent. Reference: [UUID][14] >>> print_requests.py //telemetry-ext diff --git a/acceptance/cmd/feedback/script b/acceptance/cmd/feedback/script index cc2ada0d24d..73f4f9e356f 100644 --- a/acceptance/cmd/feedback/script +++ b/acceptance/cmd/feedback/script @@ -36,6 +36,7 @@ musterr trace $CLI feedback --target dev "hello" title "Input errors are reported before authenticating" musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN $CLI feedback " " musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent $CLI feedback --run-id abc "hello" +musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent $CLI feedback --yes=false "hello" title "General CLI feedback" trace $CLI feedback "the CLI should support ..." @@ -49,6 +50,11 @@ title "An AI agent can receive a JSON receipt" trace AGENT=test-agent $CLI feedback "agent JSON feedback" -o json < /dev/null print_feedback +title "An explicit rejection from an AI agent sends nothing" +musterr trace AGENT=test-agent $CLI feedback --yes=false "hello" +musterr trace AGENT=test-agent $CLI feedback -y=false "hello" +trace find.py '^out\.requests\.txt$' --expect 0 + title "The deprecated --yes flag remains accepted" trace AGENT=test-agent $CLI feedback --yes "hello" print_feedback diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go index 6224be7f955..9c14e2a7f4e 100644 --- a/cmd/feedback/feedback.go +++ b/cmd/feedback/feedback.go @@ -29,6 +29,9 @@ const ( defaultComponent = "DeveloperEcosystem.DatabricksCLI" runIDKey = "run-id" + + legacyConfirmationFlag = "yes" + legacyConfirmationDeprecation = "confirmation is no longer required; omit this flag" ) var productComponents = map[string]string{ @@ -48,6 +51,7 @@ type options struct { command string runID string canBeContacted bool + yes bool } // submission is the validated input, checked before authenticating so that a @@ -93,8 +97,8 @@ the reference ID.` cmd.Flags().StringVar(&opts.command, "command", "", `CLI command the feedback is about, for example "bundle deploy"`) cmd.Flags().StringVar(&opts.runID, "run-id", "", "ID of the job run the feedback is about") cmd.Flags().BoolVar(&opts.canBeContacted, "can-be-contacted", false, "Allow Databricks to contact you about this feedback") - cmd.Flags().BoolP("yes", "y", false, "Deprecated; feedback is sent without confirmation") - cmd.Flags().MarkDeprecated("yes", "feedback is sent without confirmation") + cmd.Flags().BoolVarP(&opts.yes, legacyConfirmationFlag, "y", false, "Deprecated; "+legacyConfirmationDeprecation) + cmd.Flags().MarkDeprecated(legacyConfirmationFlag, legacyConfirmationDeprecation) cmd.Args = checkArgs @@ -105,6 +109,9 @@ the reference ID.` if err != nil { return err } + if cmd.Flags().Changed(legacyConfirmationFlag) && !opts.yes && useragent.AgentProvider() != "" { + return errors.New("--yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation") + } cmd.SetContext(root.SkipLoadBundle(cmd.Context())) return root.MustWorkspaceClient(cmd, args) } diff --git a/cmd/feedback/feedback_test.go b/cmd/feedback/feedback_test.go index b2a69b7c1dd..eff95d661dd 100644 --- a/cmd/feedback/feedback_test.go +++ b/cmd/feedback/feedback_test.go @@ -210,13 +210,15 @@ func TestSendWithoutConfirmation(t *testing.T) { var stdout, stderr bytes.Buffer var in io.Reader = strings.NewReader("") + var out io.Writer = &stdout var errOut io.Writer = &stderr if tt.terminal { in = cmdio.FakeTTYReader(in) + out = cmdio.FakeTTY(out) errOut = cmdio.FakeTTY(errOut) } ctx := t.Context() - ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, in, &stdout, errOut, "", "")) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, in, out, errOut, "", "")) ctx = cmdctx.SetWorkspaceClient(ctx, w) ctx = cmdctx.GenerateExecId(ctx) err = send(ctx, submission{text: "exact feedback", component: defaultComponent, origin: "cli"}) From 4e57c450429eed28a9d6851f3d3adf0ebc88c63e Mon Sep 17 00:00:00 2001 From: simonfaltum <4305831+simonfaltum@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:23:14 +0000 Subject: [PATCH 4/5] Preserve feedback declines in migration guidance Check the harness request log after declined submissions and document interactive legacy-flag behavior. --- .nextchanges/cli/feedback-agent-submissions.md | 2 +- acceptance/cmd/feedback/output.txt | 16 ++++++++-------- acceptance/cmd/feedback/script | 2 +- cmd/feedback/feedback.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.nextchanges/cli/feedback-agent-submissions.md b/.nextchanges/cli/feedback-agent-submissions.md index 1eaf3c39b6e..b92339008eb 100644 --- a/.nextchanges/cli/feedback-agent-submissions.md +++ b/.nextchanges/cli/feedback-agent-submissions.md @@ -1 +1 @@ -* Allow AI agents to submit feedback without confirmation and clarify the guidance on customer content and PII; deprecate `--yes` / `-y` while continuing to reject explicit false values from agents. ([#7795](https://github.com/databricks/cli/pull/7795)) +* Allow AI agents to submit feedback without confirmation and clarify the guidance on customer content and PII; deprecate `--yes` / `-y` and reject explicit false values from agents, including in a terminal. ([#7795](https://github.com/databricks/cli/pull/7795)) diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt index cf50f546df8..4460cbcf7a9 100644 --- a/acceptance/cmd/feedback/output.txt +++ b/acceptance/cmd/feedback/output.txt @@ -178,8 +178,8 @@ Error: feedback message must contain 1 to 10000 characters, including a visible Error: --run-id must be a positive job run ID >>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback --yes=false hello -Flag --yes has been deprecated, confirmation is no longer required; omit this flag -Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation +Flag --yes has been deprecated, omit this flag when sending feedback; skip the feedback command when declining +Error: feedback was not sent because --yes=false === General CLI feedback @@ -250,19 +250,19 @@ Thank you! Your feedback was sent. Reference: [UUID][10] === An explicit rejection from an AI agent sends nothing >>> AGENT=test-agent [CLI] feedback --yes=false hello -Flag --yes has been deprecated, confirmation is no longer required; omit this flag -Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation +Flag --yes has been deprecated, omit this flag when sending feedback; skip the feedback command when declining +Error: feedback was not sent because --yes=false >>> AGENT=test-agent [CLI] feedback -y=false hello -Flag --yes has been deprecated, confirmation is no longer required; omit this flag -Error: --yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation +Flag --yes has been deprecated, omit this flag when sending feedback; skip the feedback command when declining +Error: feedback was not sent because --yes=false ->>> find.py ^out\.requests\.txt$ --expect 0 +>>> assert_not_exists.py [OUTPUT_DIR]/out.requests.txt === The deprecated --yes flag remains accepted >>> AGENT=test-agent [CLI] feedback --yes hello -Flag --yes has been deprecated, confirmation is no longer required; omit this flag +Flag --yes has been deprecated, omit this flag when sending feedback; skip the feedback command when declining Thank you! Your feedback was sent. Reference: [UUID][14] >>> print_requests.py //telemetry-ext diff --git a/acceptance/cmd/feedback/script b/acceptance/cmd/feedback/script index 73f4f9e356f..ca5e33f2cd8 100644 --- a/acceptance/cmd/feedback/script +++ b/acceptance/cmd/feedback/script @@ -53,7 +53,7 @@ print_feedback title "An explicit rejection from an AI agent sends nothing" musterr trace AGENT=test-agent $CLI feedback --yes=false "hello" musterr trace AGENT=test-agent $CLI feedback -y=false "hello" -trace find.py '^out\.requests\.txt$' --expect 0 +trace assert_not_exists.py "$OUT_REQUESTS" title "The deprecated --yes flag remains accepted" trace AGENT=test-agent $CLI feedback --yes "hello" diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go index 9c14e2a7f4e..2465ac3acc5 100644 --- a/cmd/feedback/feedback.go +++ b/cmd/feedback/feedback.go @@ -31,7 +31,7 @@ const ( runIDKey = "run-id" legacyConfirmationFlag = "yes" - legacyConfirmationDeprecation = "confirmation is no longer required; omit this flag" + legacyConfirmationDeprecation = "omit this flag when sending feedback; skip the feedback command when declining" ) var productComponents = map[string]string{ @@ -110,7 +110,7 @@ the reference ID.` return err } if cmd.Flags().Changed(legacyConfirmationFlag) && !opts.yes && useragent.AgentProvider() != "" { - return errors.New("--yes=false declines sending feedback from an AI agent; omit --yes to send without confirmation") + return errors.New("feedback was not sent because --yes=false") } cmd.SetContext(root.SkipLoadBundle(cmd.Context())) return root.MustWorkspaceClient(cmd, args) From 2be1b3f462962b067575c1c68400bc62f8da33ab Mon Sep 17 00:00:00 2001 From: simonfaltum <4305831+simonfaltum@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:41:05 +0000 Subject: [PATCH 5/5] Clarify consent to follow-up on agent feedback --- acceptance/cmd/feedback/output.txt | 1 + cmd/feedback/feedback.go | 1 + 2 files changed, 2 insertions(+) diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt index 4460cbcf7a9..e93295b764e 100644 --- a/acceptance/cmd/feedback/output.txt +++ b/acceptance/cmd/feedback/output.txt @@ -17,6 +17,7 @@ collected. Do not include customer content, personally identifiable information is set, because this command explicitly submits feedback. AI agents can submit feedback directly without asking the user for approval. +Use --can-be-contacted only if the user explicitly agrees to be contacted. After a successful submission, tell the user that feedback was sent and include the reference ID. diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go index 2465ac3acc5..7ad42e87835 100644 --- a/cmd/feedback/feedback.go +++ b/cmd/feedback/feedback.go @@ -88,6 +88,7 @@ collected. Do not include customer content, personally identifiable information is set, because this command explicitly submits feedback. AI agents can submit feedback directly without asking the user for approval. +Use --can-be-contacted only if the user explicitly agrees to be contacted. After a successful submission, tell the user that feedback was sent and include the reference ID.` cmd.Example = ` databricks feedback --command "bundle deploy" "deploy should print which resources changed"