diff --git a/.nextchanges/cli/feedback.md b/.nextchanges/cli/feedback.md new file mode 100644 index 00000000000..b402f08a736 --- /dev/null +++ b/.nextchanges/cli/feedback.md @@ -0,0 +1 @@ +* Add the `databricks feedback` command to send feedback about the CLI to Databricks. ([#7724](https://github.com/databricks/cli/pull/7724)) diff --git a/acceptance/cmd/feedback/broken-bundle/databricks.yml b/acceptance/cmd/feedback/broken-bundle/databricks.yml new file mode 100644 index 00000000000..c8138b9f3cb --- /dev/null +++ b/acceptance/cmd/feedback/broken-bundle/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: broken-bundle + +# Two targets both marked default is an invalid bundle. Feedback does not use +# the bundle, so this error must not stop it. +targets: + dev: + default: true + dev2: + default: true diff --git a/acceptance/cmd/feedback/out.test.toml b/acceptance/cmd/feedback/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/feedback/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt new file mode 100644 index 00000000000..eff9b579f70 --- /dev/null +++ b/acceptance/cmd/feedback/output.txt @@ -0,0 +1,232 @@ + +=== Feedback about a product command is routed to its team + +>>> [CLI] feedback --command air run --run-id 123 --can-be-contacted logs omitted the failure reason +Thank you! Your feedback was sent. Reference: [UUID][0] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][0]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][1]", + "parent_ui": "ModelTraining.AIRuntime", + "origin": "air_run", + "can_be_contacted": true, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback run-id/123", + "text": "logs omitted the failure reason" + } + } +} + +=== A command group, with a JSON receipt + +>>> [CLI] feedback --command bundle deploy should print which resources changed -o json +{ + "received": true, + "feedback_id": "[UUID][2]" +} + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][2]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][3]", + "parent_ui": "DeveloperEcosystem.CoreDabs", + "origin": "bundle", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "deploy should print which resources changed" + } + } +} + +=== Feedback can name a nested command + +>>> [CLI] feedback --command bundle deploy deploy is slow +Thank you! Your feedback was sent. Reference: [UUID][4] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][4]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][5]", + "parent_ui": "DeveloperEcosystem.CoreDabs", + "origin": "bundle_deploy", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "deploy is slow" + } + } +} + +=== A command without a product team goes to the CLI team + +>>> [CLI] feedback --command no-such-command hello +Thank you! Your feedback was sent. Reference: [UUID][6] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][6]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][7]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "no-such-command", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "hello" + } + } +} + +=== Validation errors send nothing + +>>> [CLI] feedback +Error: feedback message must contain 1 to 10000 characters, including a visible one + +>>> [CLI] feedback +Error: accepts 1 arg(s), received 0 + +Usage: + databricks feedback MESSAGE [flags] + +Examples: + databricks feedback --command "bundle deploy" "deploy should print which resources changed" + databricks feedback --command "air run" --run-id 123 "logs omitted the failure reason" + +Flags: + --can-be-contacted[=true|false] Allow Databricks to contact you about this feedback + --command string CLI command the feedback is about, for example "bundle deploy" + -h, --help[=true|false] help for feedback + --run-id string ID of the job run the feedback is about + -y, --yes[=true|false] Confirm sending feedback written by an AI agent + +Global Flags: + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + + +>>> [CLI] feedback deploy is slow +Error: the message must be one argument, but received 3; put quotes around the whole message, and around --command if it has several words + +>>> [CLI] feedback --command bundle deploy deploy is slow +Error: the message must be one argument, but received 2; put quotes around the whole message, and around --command if it has several words + +>>> [CLI] feedback --command bundle deploy -t dev hello +Error: invalid --command "bundle deploy -t dev"; it takes a command path such as "bundle deploy", without the databricks prefix, flags or arguments + +>>> [CLI] feedback --run-id abc hello +Error: --run-id must be a positive job run ID + +>>> [CLI] feedback --target dev hello +Error: --target selects a bundle target, which feedback does not use; use --profile to choose the workspace + +=== Input errors are reported before authenticating + +>>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN [CLI] feedback +Error: feedback message must contain 1 to 10000 characters, including a visible one + +>>> env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent [CLI] feedback hello +Error: feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message + +Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it + +=== General CLI feedback + +>>> [CLI] feedback the CLI should support ... +Thank you! Your feedback was sent. Reference: [UUID][8] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][8]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][9]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "the CLI should support ..." + } + } +} + +=== An AI agent needs the user's confirmation + +>>> AGENT=test-agent [CLI] feedback hello +Error: feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message + +Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it + +=== An AI agent with --yes + +>>> AGENT=test-agent [CLI] feedback --yes hello +Thank you! Your feedback was sent. Reference: [UUID][10] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][10]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][11]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback agent/test-agent", + "text": "hello" + } + } +} + +=== Feedback is sent even with telemetry disabled + +>>> DATABRICKS_CLI_DISABLE_TELEMETRY=1 [CLI] feedback telemetry is disabled +Thank you! Your feedback was sent. Reference: [UUID][12] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][12]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][13]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "telemetry is disabled" + } + } +} + +=== A broken bundle does not block feedback + +>>> withdir broken-bundle [CLI] feedback bundle config does not block feedback +Thank you! Your feedback was sent. Reference: [UUID][14] + +>>> print_requests.py //telemetry-ext +{ + "frontend_log_event_id": "[UUID][14]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][15]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "bundle config does not block feedback" + } + } +} diff --git a/acceptance/cmd/feedback/script b/acceptance/cmd/feedback/script new file mode 100644 index 00000000000..deed5673fc3 --- /dev/null +++ b/acceptance/cmd/feedback/script @@ -0,0 +1,54 @@ +# Print the feedback event, keeping only user agent entries that carry feedback +# context; the rest depend on the runner. +print_feedback() { + trace print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson + | .entry.fast_feedback_log.user_agent |= (split(" ") | map(select(test("^(cli|cmd|run-id|agent)/"))) | join(" "))' +} + +title "Feedback about a product command is routed to its team" +trace $CLI feedback --command "air run" --run-id 123 --can-be-contacted " logs omitted the failure reason " +print_feedback + +title "A command group, with a JSON receipt" +trace $CLI feedback --command bundle "deploy should print which resources changed" -o json +print_feedback + +title "Feedback can name a nested command" +trace $CLI feedback --command "bundle deploy" "deploy is slow" +print_feedback + +title "A command without a product team goes to the CLI team" +trace $CLI feedback --command "no-such-command" "hello" +print_feedback + +title "Validation errors send nothing" +musterr trace $CLI feedback " " +musterr trace $CLI feedback +musterr trace $CLI feedback deploy is slow +musterr trace $CLI feedback --command bundle deploy "deploy is slow" +musterr trace $CLI feedback --command "bundle deploy -t dev" "hello" +musterr trace $CLI feedback --run-id abc "hello" +musterr trace $CLI feedback --target dev "hello" + +title "Input errors are reported before authenticating" +musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN $CLI feedback " " +musterr trace env -u DATABRICKS_HOST -u DATABRICKS_TOKEN AGENT=test-agent $CLI feedback "hello" + +title "General CLI feedback" +trace $CLI feedback "the CLI should support ..." +print_feedback + +title "An AI agent needs the user's confirmation" +musterr trace AGENT=test-agent $CLI feedback "hello" + +title "An AI agent with --yes" +trace AGENT=test-agent $CLI feedback --yes "hello" +print_feedback + +title "Feedback is sent even with telemetry disabled" +trace DATABRICKS_CLI_DISABLE_TELEMETRY=1 $CLI feedback "telemetry is disabled" +print_feedback + +title "A broken bundle does not block feedback" +trace withdir broken-bundle $CLI feedback "bundle config does not block feedback" +print_feedback diff --git a/acceptance/cmd/feedback/test.toml b/acceptance/cmd/feedback/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/cmd/feedback/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 9a63360e10c..ae4f221437f 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -185,6 +185,7 @@ Domains Developer Tools air Run and manage AI Runtime workloads on serverless GPU compute bundle Declarative Automation Bundles let you express data/AI/analytics projects as code. + feedback Send feedback about the Databricks CLI to Databricks sandbox *Beta* Manage Databricks Sandbox environments sync Synchronize a local directory to a workspace directory diff --git a/cmd/cmd.go b/cmd/cmd.go index 251058b0127..538a2fb9a26 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/cmd/completion" "github.com/databricks/cli/cmd/configure" "github.com/databricks/cli/cmd/experimental" + "github.com/databricks/cli/cmd/feedback" "github.com/databricks/cli/cmd/fs" "github.com/databricks/cli/cmd/labs" "github.com/databricks/cli/cmd/pipelines" @@ -102,6 +103,7 @@ func topLevelSpecs() []root.CommandSpec { bundle.Spec, cache.Spec, experimental.Spec, + feedback.Spec, psql.Spec, configure.Spec, fs.Spec, diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go new file mode 100644 index 00000000000..31cbfdc3b52 --- /dev/null +++ b/cmd/feedback/feedback.go @@ -0,0 +1,364 @@ +package feedback + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + "uuid" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/agent" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/databrickscfg/profile" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/useragent" + "github.com/spf13/cobra" +) + +const ( + messageLimit = 10000 + // commandLimit is well above the longest command path, about 80 characters. + commandLimit = 200 + sendTimeout = 30 * time.Second + + defaultComponent = "DeveloperEcosystem.DatabricksCLI" + runIDKey = "run-id" +) + +var productComponents = map[string]string{ + "air": "ModelTraining.AIRuntime", + "bundle": "DeveloperEcosystem.CoreDabs", +} + +const receiptTemplate = `Thank you! Your feedback was sent. Reference: {{.FeedbackID}} +` + +type receipt struct { + Received bool `json:"received"` + FeedbackID string `json:"feedback_id"` +} + +type options struct { + command string + runID string + canBeContacted bool + yes bool +} + +// submission is the validated input, checked before authenticating so that a +// mistake is reported without a login prompt or a network call. +type submission struct { + text string + command string + component string + origin string + runID string + canBeContacted bool +} + +// Spec declares the feedback command. +var Spec = root.CommandSpec{ + Use: "feedback MESSAGE", + Short: "Send feedback about the Databricks CLI to Databricks", + GroupID: "development", + Build: buildCommand, +} + +func buildCommand(ctx context.Context, cmd *cobra.Command) { + cmd.Long = `Send feedback about the Databricks CLI to Databricks. + +Describe the command you ran, what happened, what you expected, and any +workaround. Use --command to name the command the feedback is about so it +reaches the team that owns it. + +The message is sent with the workspace and user you are authenticated as and +the CLI user agent, which includes the CLI version, operating system and, if +detected, the AI agent running the CLI. No logs, files or transcripts are +collected. Do not include credentials or sensitive data. Follow-up contact is +opt-in with --can-be-contacted. Feedback is sent even when +DATABRICKS_CLI_DISABLE_TELEMETRY is set, because you asked to send it. + +AI agents: submit feedback only with the user's approval. When an agent is +detected, the command asks for confirmation, or requires --yes when no terminal +is attached. Show the user the exact message and the flags you pass with it, +such as --can-be-contacted, and pass --yes only after the user approves it.` + cmd.Example = ` databricks feedback --command "bundle deploy" "deploy should print which resources changed" + databricks feedback --command "air run" --run-id 123 "logs omitted the failure reason"` + + var opts options + cmd.Flags().StringVar(&opts.command, "command", "", `CLI command the feedback is about, for example "bundle deploy"`) + cmd.Flags().StringVar(&opts.runID, "run-id", "", "ID of the job run the feedback is about") + cmd.Flags().BoolVar(&opts.canBeContacted, "can-be-contacted", false, "Allow Databricks to contact you about this feedback") + cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, "Confirm sending feedback written by an AI agent") + + cmd.Args = checkArgs + + var s submission + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + var err error + s, err = newSubmission(cmd, args[0], &opts) + if err != nil { + return err + } + if err := checkApproval(cmd.Context(), opts.yes); err != nil { + return err + } + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + return root.MustWorkspaceClient(cmd, args) + } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + return send(cmd.Context(), s, opts.yes) + } +} + +// checkArgs requires the message as one argument. Several arguments mean that +// the message, or a --command with several words, was not quoted. The error +// says what to quote rather than printing a command to paste, which would drop +// the user's other flags and depend on their shell's quoting. +func checkArgs(cmd *cobra.Command, args []string) error { + if len(args) > 1 { + return fmt.Errorf("the message must be one argument, but received %d; put quotes around the whole message, and around --command if it has several words", len(args)) + } + return root.ExactArgs(1)(cmd, args) +} + +func newSubmission(cmd *cobra.Command, message string, opts *options) (submission, error) { + // Feedback never loads the bundle, so a target would be ignored. + for _, name := range []string{"target", "environment"} { + if cmd.Flag(name).Changed { + return submission{}, fmt.Errorf("--%s selects a bundle target, which feedback does not use; use --profile to choose the workspace", name) + } + } + text, err := validateMessage(message) + if err != nil { + return submission{}, err + } + if utf8.RuneCountInString(opts.command) > commandLimit { + return submission{}, fmt.Errorf("--command must be at most %d characters", commandLimit) + } + component, origin, err := resolveCommand(opts.command) + if err != nil { + return submission{}, err + } + s := submission{ + text: text, + command: strings.Join(strings.Fields(opts.command), " "), + component: component, + origin: origin, + canBeContacted: opts.canBeContacted, + } + if opts.runID != "" { + id, err := strconv.ParseInt(opts.runID, 10, 64) + if err != nil || id <= 0 { + return submission{}, errors.New("--run-id must be a positive job run ID") + } + s.runID = strconv.FormatInt(id, 10) + } + return s, nil +} + +func send(ctx context.Context, s submission, yes bool) error { + if s.runID != "" { + ctx = useragent.InContext(ctx, runIDKey, s.runID) + } + if provider := useragent.AgentProvider(); provider != "" { + // The SDK adds the agent to HTTP request headers, but the feedback record + // needs it too so triage can distinguish agent reports. + ctx = useragent.InContext(ctx, useragent.AgentKey, provider) + } + cfg := cmdctx.WorkspaceClient(ctx).Config + if needsApproval(yes) { + confirmed, err := confirm(ctx, s, workspaceLabel(cfg)) + if err != nil { + return err + } + if !confirmed { + // confirm told the user; the exit code tells the agent that ran the + // command that nothing was sent. + return root.ErrAlreadyPrinted + } + } + + id := uuid.New().String() + sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) + defer cancel() + err := telemetry.Send(sendCtx, cfg, []protos.FrontendLog{{ + FrontendLogEventID: id, + Entry: protos.FrontendLogEntry{ + FastFeedbackLog: &protos.FastFeedbackLog{ + ReferenceEventID: cmdctx.ExecId(ctx), + ParentUI: s.component, + Origin: s.origin, + CanBeContacted: s.canBeContacted, + UIVersions: "cli:" + build.GetInfo().Version, + UserAgent: useragent.FromContext(ctx), + Text: s.text, + }, + }, + }}) + if err != nil { + if telemetry.NotSent(err) { + return fmt.Errorf("feedback %s was not sent: %w", id, err) + } + // The upload may have been accepted before the error, so retrying can + // file the same feedback twice. + return fmt.Errorf("feedback %s could not be confirmed as sent: %w", id, err) + } + + return cmdio.RenderWithTemplate(ctx, receipt{Received: true, FeedbackID: id}, "", receiptTemplate) +} + +func validateMessage(message string) (string, error) { + text := strings.TrimSpace(message) + if utf8.RuneCountInString(text) > messageLimit || !strings.ContainsFunc(text, visible) { + return "", fmt.Errorf("feedback message must contain 1 to %d characters, including a visible one", messageLimit) + } + return text, nil +} + +// resolveCommand returns the component and origin for --command. The first word +// picks the owning team. The command is not looked up, which would mean +// building the command tree; its words are only limited to command-name +// characters, which keeps flags, paths and control sequences out of origin and +// the approval prompt. The origin joins the words with underscores, as in +// bundle_deploy. +func resolveCommand(command string) (string, string, error) { + words := strings.Fields(command) + if len(words) == 0 { + return defaultComponent, "cli", nil + } + for _, word := range words { + if !isCommandWord(word) { + return "", "", fmt.Errorf(`invalid --command %q; it takes a command path such as "bundle deploy", without the databricks prefix, flags or arguments`, command) + } + } + component := defaultComponent + if productComponent, ok := productComponents[words[0]]; ok { + component = productComponent + } + return component, strings.Join(words, "_"), nil +} + +// isCommandWord reports whether word can be part of a command path: lowercase +// letters, digits and dashes, not starting with a dash. +func isCommandWord(word string) bool { + if word == "" || word[0] == '-' { + return false + } + for _, r := range word { + if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' { + return false + } + } + return true +} + +// needsApproval reports whether the user must approve the feedback before it is +// sent: an AI agent is running the CLI and did not pass --yes. +func needsApproval(yes bool) bool { + return useragent.AgentProvider() != "" && !yes +} + +// checkApproval returns an error when the user must approve the feedback but +// there is no terminal to ask in. It needs only the terminal, so it runs before +// authenticating. confirm shows the summary on stderr and reads a plain line +// from stdin, so it works where cmdio.IsPromptSupported says no (TERM=dumb, +// MSYS2 pipes). +func checkApproval(ctx context.Context, yes bool) error { + if !needsApproval(yes) || (cmdio.IsInputTTY(ctx) && cmdio.IsErrorTTY(ctx)) { + return nil + } + return errors.New("feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message\n\n" + + agent.AgentNote("show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it")) +} + +// confirm asks the user to approve feedback an AI agent is about to send. It +// returns false without error if the user declines. +func confirm(ctx context.Context, s submission, workspace string) (bool, error) { + cmdio.LogString(ctx, "The following feedback will be sent to Databricks:\n\n"+summary(s, workspace)) + confirmed, err := cmdio.AskYesOrNo(ctx, "Send this feedback?") + switch { + case errors.Is(err, io.EOF): + // Ctrl-D ends the input without an answer or a newline. + cmdio.LogString(ctx, "\nFeedback not sent.") + return false, nil + case err != nil: + return false, err + case !confirmed: + cmdio.LogString(ctx, "Feedback not sent.") + } + return confirmed, nil +} + +// workspaceLabel names the workspace feedback is filed in. A unified host +// serves several workspaces, so there the host alone does not identify it. +func workspaceLabel(cfg *config.Config) string { + if id := cfg.WorkspaceID; id != "" && id != profile.WorkspaceIDNone { + return fmt.Sprintf("%s (workspace ID %s)", cfg.Host, id) + } + return cfg.Host +} + +// summary returns the message, the workspace it is filed in and the choices +// sent with it, for the user to approve. +func summary(s submission, workspace string) string { + var b strings.Builder + // Indent the message so that none of its lines can pass for one of the + // fields below it. + b.WriteString(" " + strings.ReplaceAll(escapeInvisible(s.text), "\n", "\n ")) + b.WriteString("\n\n") + fmt.Fprintf(&b, "Workspace: %s\n", workspace) + if s.command != "" { + fmt.Fprintf(&b, "Command: %s\n", s.command) + } + if s.runID != "" { + fmt.Fprintf(&b, "Run ID: %s\n", s.runID) + } + contact := "no" + if s.canBeContacted { + contact = "yes" + } + fmt.Fprintf(&b, "Databricks may contact you: %s\n", contact) + return b.String() +} + +// escapeInvisible escapes the runes that invisible reports, so none of them can +// hide, reorder or overwrite part of the message in the terminal. +func escapeInvisible(text string) string { + var b strings.Builder + for _, r := range text { + if invisible(r) { + quoted := strconv.QuoteRuneToASCII(r) + b.WriteString(quoted[1 : len(quoted)-1]) + continue + } + b.WriteRune(r) + } + return b.String() +} + +// invisible reports whether a terminal would not show r as written, other than +// a newline or tab: characters that are not graphic, such as escape sequences, +// bidi overrides and zero-width characters, and the variation selectors and +// fillers that Unicode marks as default ignorable, which can carry hidden text. +func invisible(r rune) bool { + if r == '\n' || r == '\t' { + return false + } + return !unicode.IsGraphic(r) || unicode.In(r, unicode.Variation_Selector, unicode.Other_Default_Ignorable_Code_Point) +} + +// visible reports whether r shows up as a mark in the terminal. +func visible(r rune) bool { + return !unicode.IsSpace(r) && !invisible(r) +} diff --git a/cmd/feedback/feedback_test.go b/cmd/feedback/feedback_test.go new file mode 100644 index 00000000000..e9ff919ff9d --- /dev/null +++ b/cmd/feedback/feedback_test.go @@ -0,0 +1,364 @@ +package feedback + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateMessage(t *testing.T) { + tests := []struct { + name string + message string + want string + wantErr bool + }{ + {name: "trimmed", message: " feedback\n", want: "feedback"}, + {name: "maximum runes", message: strings.Repeat("é", messageLimit), want: strings.Repeat("é", messageLimit)}, + {name: "blank", message: " \n", wantErr: true}, + {name: "only invisible characters", message: "\u200b\ufe0f\u3164", wantErr: true}, + {name: "visible after invisible", message: "\u200bx", want: "\u200bx"}, + {name: "too long", message: strings.Repeat("x", messageLimit+1), wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateMessage(tt.message) + if tt.wantErr { + assert.EqualError(t, err, messageError) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +const ( + messageError = "feedback message must contain 1 to 10000 characters, including a visible one" + runIDError = "--run-id must be a positive job run ID" +) + +const testWorkspace = "https://workspace.test" + +// newTestFeedbackCommand returns the feedback command under a root with the +// root's bundle target flags. +func newTestFeedbackCommand() *cobra.Command { + rootCmd := &cobra.Command{Use: "databricks"} + rootCmd.PersistentFlags().String("target", "", "") + rootCmd.PersistentFlags().String("environment", "", "") + feedbackCmd := &cobra.Command{Use: "feedback"} + rootCmd.AddCommand(feedbackCmd) + return feedbackCmd +} + +func TestCheckArgs(t *testing.T) { + tests := []struct { + name string + args []string + wantErr string + }{ + {name: "one message", args: []string{"deploy is slow"}}, + {name: "no message", wantErr: "accepts 1 arg(s), received 0"}, + { + name: "unquoted message or --command", + args: []string{"deploy", "is", "slow"}, + wantErr: "the message must be one argument, but received 3; put quotes around the whole message, and around --command if it has several words", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkArgs(newTestFeedbackCommand(), tt.args) + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestWorkspaceLabel(t *testing.T) { + tests := []struct { + name string + workspaceID string + want string + }{ + {name: "workspace host", want: testWorkspace}, + {name: "unified host", workspaceID: "123", want: testWorkspace + " (workspace ID 123)"}, + {name: "no workspace sentinel", workspaceID: "none", want: testWorkspace}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, workspaceLabel(&config.Config{Host: testWorkspace, WorkspaceID: tt.workspaceID})) + }) + } +} + +func TestResolveCommand(t *testing.T) { + tests := []struct { + name string + command string + wantComponent string + wantOrigin string + wantErr bool + }{ + {name: "general", wantComponent: defaultComponent, wantOrigin: "cli"}, + {name: "blank", command: " ", wantComponent: defaultComponent, wantOrigin: "cli"}, + {name: "air subcommand", command: "air run", wantComponent: "ModelTraining.AIRuntime", wantOrigin: "air_run"}, + {name: "bundle group", command: "bundle", wantComponent: "DeveloperEcosystem.CoreDabs", wantOrigin: "bundle"}, + {name: "bundle subcommand", command: "bundle deploy", wantComponent: "DeveloperEcosystem.CoreDabs", wantOrigin: "bundle_deploy"}, + {name: "unchecked subcommand", command: "bundle not-a-command", wantComponent: "DeveloperEcosystem.CoreDabs", wantOrigin: "bundle_not-a-command"}, + {name: "other command", command: "jobs list", wantComponent: defaultComponent, wantOrigin: "jobs_list"}, + {name: "unchecked top level", command: "bundel deploy", wantComponent: defaultComponent, wantOrigin: "bundel_deploy"}, + {name: "flag", command: "bundle deploy -t dev", wantErr: true}, + {name: "path argument", command: "bundle sync ./src", wantErr: true}, + {name: "uppercase", command: "Bundle deploy", wantErr: true}, + {name: "control characters", command: "bundle \x1b[8mdeploy", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + component, origin, err := resolveCommand(tt.command) + if tt.wantErr { + assert.EqualError(t, err, fmt.Sprintf(`invalid --command %q; it takes a command path such as "bundle deploy", without the databricks prefix, flags or arguments`, tt.command)) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantComponent, component) + assert.Equal(t, tt.wantOrigin, origin) + }) + } +} + +func TestNewSubmission(t *testing.T) { + tests := []struct { + name string + flags []string + opts options + message string + want submission + wantErr string + }{ + { + name: "all fields", + opts: options{command: "bundle deploy", runID: "+00123", canBeContacted: true}, + message: " slow ", + want: submission{ + text: "slow", command: "bundle deploy", component: "DeveloperEcosystem.CoreDabs", + origin: "bundle_deploy", runID: "123", canBeContacted: true, + }, + }, + {name: "zero run ID", opts: options{runID: "0"}, message: "x", wantErr: runIDError}, + {name: "non-numeric run ID", opts: options{runID: "abc"}, message: "x", wantErr: runIDError}, + {name: "blank message", message: " ", wantErr: messageError}, + { + name: "long command", opts: options{command: "bundle " + strings.Repeat("a", commandLimit)}, message: "x", + wantErr: fmt.Sprintf("--command must be at most %d characters", commandLimit), + }, + { + name: "target", flags: []string{"--target", "dev"}, message: "x", + wantErr: "--target selects a bundle target, which feedback does not use; use --profile to choose the workspace", + }, + { + name: "deprecated environment", flags: []string{"--environment", "dev"}, message: "x", + wantErr: "--environment selects a bundle target, which feedback does not use; use --profile to choose the workspace", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + feedbackCmd := newTestFeedbackCommand() + require.NoError(t, feedbackCmd.ParseFlags(tt.flags)) + got, err := newSubmission(feedbackCmd, tt.message, &tt.opts) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestSummary(t *testing.T) { + tests := []struct { + name string + s submission + want string + }{ + { + name: "message only", + s: submission{text: "line one\n\tline two"}, + want: " line one\n \tline two\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", + }, + { + name: "all fields", + s: submission{text: "slow", command: "air run", runID: "42", canBeContacted: true}, + want: " slow\n\nWorkspace: https://workspace.test\nCommand: air run\nRun ID: 42\nDatabricks may contact you: yes\n", + }, + { + name: "message lines cannot pass for fields", + s: submission{text: "slow\nDatabricks may contact you: no", canBeContacted: true}, + want: " slow\n Databricks may contact you: no\n\nWorkspace: https://workspace.test\nDatabricks may contact you: yes\n", + }, + { + name: "control characters cannot hide text", + s: submission{text: "great\x1b[8m hidden\x1b[0m\r\u0085"}, + want: ` great\x1b[8m hidden\x1b[0m\r\u0085` + "\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", + }, + { + name: "format characters cannot reorder or hide text", + s: submission{text: "a\u202eb\u200bc\U000E0041 d\u00a0é"}, + want: ` a\u202eb\u200bc\U000e0041 d` + "\u00a0é\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", + }, + { + name: "variation selectors and fillers cannot carry hidden text", + s: submission{text: "ok👍\ufe0f\U000E0100\u3164\u034f e\u0301"}, + want: ` ok👍\ufe0f\U000e0100\u3164\u034f ` + "e\u0301\n\nWorkspace: https://workspace.test\nDatabricks may contact you: no\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, summary(tt.s, testWorkspace)) + }) + } +} + +func TestConfirm(t *testing.T) { + tests := []struct { + name string + answer string + wantConfirmed bool + wantStderr string + }{ + {name: "accepted", answer: "y\n", wantConfirmed: true, wantStderr: "Send this feedback? [y/N]: "}, + {name: "declined", answer: "\n", wantStderr: "Feedback not sent."}, + {name: "input ended", answer: "", wantStderr: "Send this feedback? [y/N]: \nFeedback not sent."}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(tt.answer), io.Discard, &stderr)) + confirmed, err := confirm(ctx, submission{text: "exact feedback"}, testWorkspace) + require.NoError(t, err) + assert.Equal(t, tt.wantConfirmed, confirmed) + assert.Contains(t, stderr.String(), "The following feedback will be sent to Databricks:\n\n exact feedback\n\nWorkspace: "+testWorkspace+"\nDatabricks may contact you: no\n\n") + assert.Contains(t, stderr.String(), tt.wantStderr) + }) + } +} + +func TestCheckApproval(t *testing.T) { + const refusal = "feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message\n\n" + + "Note for AI agents (test-agent): show the user the exact feedback message and the flags you pass with it, such as --can-be-contacted, and pass --yes only after the user approves sending it" + tests := []struct { + name string + agent string + yes bool + terminal bool + redirectStderr bool + envKey string + envValue string + wantErr string + }{ + {name: "human without a terminal"}, + {name: "agent with --yes", agent: "test-agent", yes: true}, + {name: "agent with a terminal", agent: "test-agent", terminal: true}, + {name: "agent with a terminal and NO_COLOR", agent: "test-agent", terminal: true, envKey: "NO_COLOR", envValue: "1"}, + {name: "agent with a dumb terminal", agent: "test-agent", terminal: true, envKey: "TERM", envValue: "dumb"}, + {name: "agent with stderr redirected", agent: "test-agent", terminal: true, redirectStderr: true, wantErr: refusal}, + {name: "agent without a terminal", agent: "test-agent", wantErr: refusal}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutil.SetAgent(t, tt.agent) + ctx := t.Context() + if tt.envKey != "" { + ctx = env.Set(ctx, tt.envKey, tt.envValue) + } + var in io.Reader = strings.NewReader("") + if tt.terminal { + in = cmdio.FakeTTYReader(in) + } + stderr := io.Discard + if !tt.redirectStderr { + stderr = cmdio.FakeTTY(stderr) + } + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, in, io.Discard, stderr, "", "")) + err := checkApproval(ctx, tt.yes) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + return + } + assert.NoError(t, err) + }) + } +} + +func TestSendAsksBeforeSending(t *testing.T) { + tests := []struct { + name string + answer string + wantSent []string + wantErr error + }{ + {name: "approved", answer: "y\n", wantSent: []string{"exact feedback"}}, + {name: "declined", answer: "n\n", wantErr: root.ErrAlreadyPrinted}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Start the server and the client before SetAgent clears the + // environment: on Windows, sockets need SystemRoot. + server := testserver.New(t) + var sent []string + server.Handle(http.MethodPost, "/telemetry-ext", func(req testserver.Request) any { + // The handler runs on the server's goroutine, so it must not + // call require. + var body telemetry.RequestBody + assert.NoError(t, json.Unmarshal(req.Body, &body)) + for _, raw := range body.ProtoLogs { + var entry protos.FrontendLog + if assert.NoError(t, json.Unmarshal([]byte(raw), &entry)) { + sent = append(sent, entry.Entry.FastFeedbackLog.Text) + } + } + return telemetry.ResponseBody{NumProtoSuccess: int64(len(body.ProtoLogs))} + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + testutil.SetAgent(t, "test-agent") + + var stdout, stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewTestIO(strings.NewReader(tt.answer), &stdout, &stderr)) + ctx = cmdctx.SetWorkspaceClient(ctx, w) + ctx = cmdctx.GenerateExecId(ctx) + err = send(ctx, submission{text: "exact feedback", component: defaultComponent, origin: "cli"}, false) + + assert.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.wantSent, sent) + assert.Contains(t, stderr.String(), " exact feedback\n") + if tt.wantErr == nil { + assert.Contains(t, stdout.String(), "Thank you! Your feedback was sent.") + } else { + assert.Empty(t, stdout.String()) + } + }) + } +} diff --git a/cmd/root/fuzz_panic_test.go b/cmd/root/fuzz_panic_test.go index 1fc8fd9c17d..a52815ffb63 100644 --- a/cmd/root/fuzz_panic_test.go +++ b/cmd/root/fuzz_panic_test.go @@ -209,6 +209,7 @@ func isAutoGenerated(leaf leafCommand) bool { "completion": true, "configure": true, "experimental": true, + "feedback": true, "labs": true, "sandbox": true, "pipelines": true, diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index 9045dddcdf1..1da98ba9a1d 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -333,6 +333,7 @@ var hintExemptCommands = []string{ "configure", "current-user me", "version", + "feedback", // A one-shot report: the agent has no task to apply the skills to. "completion", "help", cobra.ShellCompRequestCmd, diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 50cdca4afcd..ad0c04a6a53 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -260,6 +260,7 @@ func TestIsHintExemptCommand(t *testing.T) { }{ {"aitools install", true}, {"experimental aitools skills install", true}, + {"feedback", true}, {"auth login", true}, {"auth token", true}, {"configure", true}, diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go index 5a0385d664a..7ca1efb3cb6 100644 --- a/libs/cmdio/io.go +++ b/libs/cmdio/io.go @@ -92,6 +92,13 @@ func IsInputTTY(ctx context.Context) bool { return c.capabilities.stdinIsTTY } +// IsErrorTTY reports whether stderr is a terminal, so that the user sees the +// messages and questions written there. Like IsInputTTY, it ignores TERM=dumb. +func IsErrorTTY(ctx context.Context) bool { + c := fromContext(ctx) + return c.capabilities.stderrIsTTY +} + // IsPagerSupported reports whether stdin, stdout, and stderr are all interactive // terminals. This is the requirement for a full-screen or navigable output // program: unlike IsPromptSupported it also checks stdout, so it returns false diff --git a/libs/cmdio/io_test.go b/libs/cmdio/io_test.go index b926dec3cd7..0dbd0462cbf 100644 --- a/libs/cmdio/io_test.go +++ b/libs/cmdio/io_test.go @@ -52,6 +52,14 @@ func TestIsInputTTY(t *testing.T) { assert.False(t, IsInputTTY(ctx)) } +func TestIsErrorTTY(t *testing.T) { + ctx := InContext(t.Context(), &cmdIO{capabilities: Capabilities{stderrIsTTY: true, dumb: true}}) + assert.True(t, IsErrorTTY(ctx)) + + ctx = InContext(t.Context(), &cmdIO{capabilities: Capabilities{stdinIsTTY: true}}) + assert.False(t, IsErrorTTY(ctx)) +} + func TestHasTerminal(t *testing.T) { for _, tc := range []struct { name string diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index aac2063fbe5..3a4f6d7edf6 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "net/http/httptrace" "strconv" "sync/atomic" "time" @@ -114,13 +115,9 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { l.logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec } - protoLogs := make([]string, len(l.logs)) - for i, log := range l.logs { - b, err := json.Marshal(log) - if err != nil { - return fmt.Errorf("failed to marshal log: %w", err) - } - protoLogs[i] = string(b) + protoLogs, err := marshalLogs(l.logs) + if err != nil { + return err } if !cmdctx.HasConfigUsed(ctx) { @@ -150,7 +147,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { - return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running) + return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running, true) }) if err != nil { return fmt.Errorf("failed to create API client: %w", err) @@ -288,8 +285,9 @@ func within[T any](ctx context.Context, running *atomic.Int32, f func() (T, erro } // newAPIClient is client.New with authenticate as the auth visitor. The SDK -// runs the auth visitor on every request, including its own retries. -func newAPIClient(cfg *config.Config, running *atomic.Int32) (*client.DatabricksClient, error) { +// runs the auth visitor on every request, including its own retries. Without +// retries, the client makes a single attempt. +func newAPIClient(cfg *config.Config, running *atomic.Int32, retries bool) (*client.DatabricksClient, error) { clientCfg, err := config.HTTPClientConfigFromConfig(cfg) if err != nil { return nil, err @@ -297,6 +295,15 @@ func newAPIClient(cfg *config.Config, running *atomic.Int32) (*client.Databricks clientCfg.AuthVisitor = func(r *http.Request) error { return authenticate(r, cfg, running) } + if !retries { + // The SDK retries some connection errors regardless of ErrorRetriable, + // so a minimal retry window is needed to stop after the first attempt. + // The attempt itself runs on the caller's context, so the window only + // ends the retrying; TestSendMakesOneAttempt fails if that changes. + clientCfg.ErrorRetriable = func(context.Context, error) bool { return false } + clientCfg.TransientErrors = nil + clientCfg.RetryTimeout = time.Nanosecond + } return client.NewWithClient(cfg, httpclient.NewApiClient(clientCfg)) } @@ -309,12 +316,94 @@ func authenticate(r *http.Request, cfg *config.Config, running *atomic.Int32) er return struct{}{}, cfg.Authenticate(authReq) }) if err != nil { - return fmt.Errorf("authenticating telemetry upload: %w", err) + // Authentication runs before the request is sent. + return ¬SentError{fmt.Errorf("authenticating upload: %w", err)} } r.Header = authReq.Header return nil } +// Send uploads logs in a single request and returns an error unless all of them +// were accepted. It is for events the user explicitly asked to send, so unlike +// Upload it ignores DATABRICKS_CLI_DISABLE_TELEMETRY and never retries. +func Send(ctx context.Context, cfg *config.Config, logs []protos.FrontendLog) error { + protoLogs, err := marshalLogs(logs) + if err != nil { + return ¬SentError{err} + } + + var running atomic.Int32 + apiClient, err := within(ctx, &running, func() (*client.DatabricksClient, error) { + return newAPIClient(cfg, &running, false) + }) + if err != nil { + return ¬SentError{fmt.Errorf("failed to create API client: %w", err)} + } + + // A request is written only once its connection is set up, so a failure + // before then, such as a DNS, dial or TLS error, means nothing was sent. + var connected atomic.Bool + traceCtx := httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GotConn: func(httptrace.GotConnInfo) { connected.Store(true) }, + }) + resp, err := attempt(traceCtx, apiClient, protoLogs) + switch { + case err != nil && !connected.Load(): + return ¬SentError{err} + case err != nil: + return err + case resp.NumProtoSuccess < int64(len(protoLogs)): + return ¬AcceptedError{Accepted: resp.NumProtoSuccess, Sent: int64(len(protoLogs))} + } + return nil +} + +// notSentError marks a Send failure that happened before the request was sent, +// so the server cannot have stored the logs. +type notSentError struct { + err error +} + +func (e *notSentError) Error() string { return e.err.Error() } +func (e *notSentError) Unwrap() error { return e.err } + +// NotSent reports whether err, returned by Send, shows that the server stored +// none of the logs: the request was never sent, the server rejected it, or it +// accepted none of the logs. Any other error is ambiguous, because the server +// may have stored the logs before the error. +func NotSent(err error) bool { + if _, ok := errors.AsType[*notSentError](err); ok { + return true + } + if e, ok := errors.AsType[*notAcceptedError](err); ok { + return e.Accepted == 0 + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 +} + +// notAcceptedError is returned by Send when the server accepted fewer logs than +// were sent, so the others were not stored. +type notAcceptedError struct { + Accepted, Sent int64 +} + +func (e *notAcceptedError) Error() string { + return fmt.Sprintf("only %d of %d logs were accepted", e.Accepted, e.Sent) +} + +func marshalLogs(logs []protos.FrontendLog) ([]string, error) { + protoLogs := make([]string, len(logs)) + for i, log := range logs { + b, err := json.Marshal(log) + if err != nil { + return nil, fmt.Errorf("failed to marshal log: %w", err) + } + protoLogs[i] = string(b) + } + return protoLogs, nil +} + func attempt(ctx context.Context, apiClient *client.DatabricksClient, protoLogs []string) (*ResponseBody, error) { // Without the workspace routing header, telemetry on unified hosts is // recorded in a central shard instead of the user's workspace. diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 94673a9dd1a..4dbba27c012 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -2,7 +2,12 @@ package telemetry import ( "context" + "encoding/json" "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" "sync/atomic" "testing" "time" @@ -465,3 +470,159 @@ func TestWithinReturnsPanicAsError(t *testing.T) { assert.EqualError(t, err, "panic: boom") assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) } + +func TestNotSent(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "never sent", err: fmt.Errorf("wrapped: %w", ¬SentError{errors.New("no client")}), want: true}, + {name: "nothing accepted", err: ¬AcceptedError{Accepted: 0, Sent: 1}, want: true}, + {name: "some accepted", err: ¬AcceptedError{Accepted: 1, Sent: 2}}, + {name: "rejected", err: &apierr.APIError{StatusCode: 400}, want: true}, + {name: "server error", err: &apierr.APIError{StatusCode: 500}}, + {name: "connection reset", err: errors.New("connection reset by peer")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, NotSent(tt.err)) + }) + } +} + +func testFeedbackLog() []protos.FrontendLog { + return []protos.FrontendLog{{ + FrontendLogEventID: "id", + Entry: protos.FrontendLogEntry{ + FastFeedbackLog: &protos.FastFeedbackLog{Text: "hello"}, + }, + }} +} + +func TestSendMakesOneAttempt(t *testing.T) { + tests := []struct { + name string + numProtoSuccess int64 + statusCode int + reset bool + wantErr bool + errContains string + wantNotSent bool + }{ + {name: "all accepted", numProtoSuccess: 1}, + {name: "none accepted", wantErr: true, errContains: "only 0 of 1 logs were accepted", wantNotSent: true}, + {name: "server error", statusCode: 500, wantErr: true, errContains: "server error"}, + {name: "SDK retryable error", statusCode: 503, wantErr: true, errContains: "server error"}, + {name: "rejected", statusCode: 403, wantErr: true, errContains: "server error", wantNotSent: true}, + {name: "connection reset", reset: true, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := testserver.New(t) + var count atomic.Int32 + var body RequestBody + if tt.reset { + server.HandleRaw(http.MethodPost, "/telemetry-ext", func(w http.ResponseWriter, r *http.Request) { + count.Add(1) + conn, _, err := w.(http.Hijacker).Hijack() + if !assert.NoError(t, err) { + return + } + // A zero linger makes Close send a TCP reset, which the SDK + // retries unless retries are off. + assert.NoError(t, conn.(*net.TCPConn).SetLinger(0)) + assert.NoError(t, conn.Close()) + }) + } else { + server.Handle(http.MethodPost, "/telemetry-ext", func(req testserver.Request) any { + count.Add(1) + assert.NoError(t, json.Unmarshal(req.Body, &body)) + if tt.statusCode != 0 { + return testserver.Response{ + StatusCode: tt.statusCode, + Body: apierr.APIError{StatusCode: tt.statusCode, Message: "server error"}, + } + } + return ResponseBody{NumProtoSuccess: tt.numProtoSuccess} + }) + } + + err := Send(t.Context(), &config.Config{Host: server.URL, Token: "token"}, testFeedbackLog()) + if !tt.wantErr { + require.NoError(t, err) + } else { + require.Error(t, err) + if tt.errContains != "" { + assert.ErrorContains(t, err, tt.errContains) + } + assert.Equal(t, tt.wantNotSent, NotSent(err)) + } + assert.Equal(t, int32(1), count.Load()) + if !tt.reset { + assert.Equal(t, []string{`{"frontend_log_event_id":"id","entry":{"fast_feedback_log":{"can_be_contacted":false,"text":"hello"}}}`}, body.ProtoLogs) + } + }) + } +} + +func TestSendFailingBeforeConnectingIsNotSent(t *testing.T) { + // The client does not trust the certificate of this server. It starts + // first so that it cannot take the port of the closed listener below. + untrusted := httptest.NewTLSServer(http.NotFoundHandler()) + t.Cleanup(untrusted.Close) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + refused := "http://" + listener.Addr().String() + require.NoError(t, listener.Close()) + + // The SDK retries its host metadata lookup for minutes when it fails. + noMetadata := func(context.Context, string) (*config.HostMetadata, error) { + return nil, nil + } + + tests := []struct { + name string + host string + }{ + {name: "connection refused", host: refused}, + {name: "untrusted certificate", host: untrusted.URL}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &config.Config{Host: tt.host, Token: "token", HostMetadataResolver: noMetadata} + err := Send(t.Context(), cfg, testFeedbackLog()) + require.Error(t, err) + assert.True(t, NotSent(err), "error: %v", err) + }) + } +} + +func TestSendTimeoutBoundsTokenAcquisition(t *testing.T) { + server := testserver.New(t) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ts := auth.TokenSourceFn(func(context.Context) (*oauth2.Token, error) { + <-release + return nil, errors.New("token endpoint unavailable") + }) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + + errc := make(chan error, 1) + go func() { + errc <- Send(ctx, &config.Config{ + Host: server.URL, + Credentials: config.NewTokenSourceStrategy("blocking", ts), + }, testFeedbackLog()) + }() + select { + case err := <-errc: + assert.ErrorIs(t, err, context.DeadlineExceeded) + // The token was never fetched, so nothing was sent. + assert.True(t, NotSent(err)) + case <-time.After(10 * time.Second): + t.Fatal("Send did not return after its context deadline") + } +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index a6ae4c7cbd8..edf00b56b3c 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -20,6 +20,12 @@ native line printed since the text line last printed. So when the command that p resolves no auth (e.g. `databricks bundle schema`), no event for that line is uploaded until the text line prints again, even though the agent saw the hint. +`databricks feedback` sends one `fast_feedback_log` entry, the event the in-product feedback widget +records, instead of a `databricks_cli_log`. The user asked for it to be sent, so it is uploaded +immediately in a single attempt even when `DATABRICKS_CLI_DISABLE_TELEMETRY` is set. When an AI +agent is detected, the user must first confirm the message. Agent and run context ride in the +entry's `user_agent`. + `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED with the outcome and error category. The CLI sets no calling-agent field on it: the upload's diff --git a/libs/telemetry/protos/fast_feedback_log.go b/libs/telemetry/protos/fast_feedback_log.go new file mode 100644 index 00000000000..98ead0b5abe --- /dev/null +++ b/libs/telemetry/protos/fast_feedback_log.go @@ -0,0 +1,29 @@ +package protos + +// FastFeedbackLog corresponds to the FastFeedbackLog lumberjack proto in universe +// (proto/logs/frontend/docs-platform/fast_feedback_log.proto). It is the event the +// in-product feedback widget records, so CLI feedback is triaged the same way. +type FastFeedbackLog struct { + // ID of an event the feedback relates to. The CLI sets the execution ID of the + // feedback command itself, which its requests also carry as cmd-exec-id in the + // user agent. + ReferenceEventID string `json:"reference_event_id,omitempty"` + + // ES component the feedback is filed under (e.g. "DeveloperEcosystem.DatabricksCLI"). + ParentUI string `json:"parent_ui,omitempty"` + + // Feature within the parent UI the feedback is about. + Origin string `json:"origin,omitempty"` + + CanBeContacted bool `json:"can_be_contacted"` + + // Comma-separated "key:value" pairs of component versions. + UIVersions string `json:"ui_versions,omitempty"` + + UserAgent string `json:"user_agent,omitempty"` + + // Feedback text written by the user. Unlike other telemetry it is user-authored + // by design: the user asked to send it, and confirms it first when an AI agent + // wrote it. + Text string `json:"text,omitempty"` +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..93aad455d1d 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -9,8 +9,11 @@ type FrontendLog struct { Entry FrontendLogEntry `json:"entry,omitempty"` } +// FrontendLogEntry mirrors a oneof in the proto, so set only one field. omitzero +// keeps an unset DatabricksCliLog, a value field, out of the JSON. type FrontendLogEntry struct { - DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitempty"` + DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitzero"` + FastFeedbackLog *FastFeedbackLog `json:"fast_feedback_log,omitempty"` } type DatabricksCliLog struct {