diff --git a/.nextchanges/cli/serving-endpoints-query-dropped-fields.md b/.nextchanges/cli/serving-endpoints-query-dropped-fields.md new file mode 100644 index 00000000000..936090e087f --- /dev/null +++ b/.nextchanges/cli/serving-endpoints-query-dropped-fields.md @@ -0,0 +1 @@ +* `serving-endpoints query` now prints a warning on stderr that names the response fields its output leaves out (for example `choices[].finish_reason`), and how to see the full response with `databricks api post`. The output itself is unchanged. ([#7664](https://github.com/databricks/cli/pull/7664)) diff --git a/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/out.test.toml b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/output.txt b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/output.txt new file mode 100644 index 00000000000..f8a4f4b2b6d --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/output.txt @@ -0,0 +1,34 @@ + +=== Fields the output doesn't show are named on stderr; stdout is unchanged + +>>> [CLI] serving-endpoints query chat-endpoint --json {"messages": [{"role": "user", "content": "hi"}]} +Warning: the endpoint's response has fields that this output doesn't show: choices[].finish_reason, service_tier + +To see the full response, send the same request with: + databricks api post /serving-endpoints/chat-endpoint/invocations --json ... + +{ + "choices": [ + { + "index": 0, + "message": { + "content": "Hello", + "role": "assistant" + } + } + ], + "id": "chatcmpl-1", + "object": "chat.completion" +} + +=== No warning when nothing is dropped + +>>> [CLI] serving-endpoints query custom-endpoint --json {"dataframe_records": [{"text": "I love this"}]} +{ + "predictions": [ + { + "confidence": 0.9, + "label": "positive" + } + ] +} diff --git a/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/script b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/script new file mode 100644 index 00000000000..7baea781298 --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/script @@ -0,0 +1,5 @@ +title "Fields the output doesn't show are named on stderr; stdout is unchanged" +trace $CLI serving-endpoints query chat-endpoint --json '{"messages": [{"role": "user", "content": "hi"}]}' + +title "No warning when nothing is dropped" +trace $CLI serving-endpoints query custom-endpoint --json '{"dataframe_records": [{"text": "I love this"}]}' diff --git a/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/test.toml b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/test.toml new file mode 100644 index 00000000000..18ad3d52207 --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-dropped-fields/test.toml @@ -0,0 +1,15 @@ +Cloud = false + +# The testserver doesn't model invocations; stub a chat response and a custom +# model response. +[[Server]] +Pattern = "POST /serving-endpoints/chat-endpoint/invocations" +Response.Body = ''' +{"id": "chatcmpl-1", "object": "chat.completion", "service_tier": "default", "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Hello"}}]} +''' + +[[Server]] +Pattern = "POST /serving-endpoints/custom-endpoint/invocations" +Response.Body = ''' +{"predictions": [{"label": "positive", "confidence": 0.9}]} +''' diff --git a/cmd/workspace/serving-endpoints/query.go b/cmd/workspace/serving-endpoints/query.go index 551a674fe08..9aeff4255c0 100644 --- a/cmd/workspace/serving-endpoints/query.go +++ b/cmd/workspace/serving-endpoints/query.go @@ -3,7 +3,9 @@ package serving_endpoints import ( "bufio" "bytes" + "context" "encoding/json" + "errors" "fmt" "io" "maps" @@ -30,77 +32,89 @@ const queryStreamHelp = ` For a chat endpoint, the reply text of each chunk is in .choices[0].delta.content.` -// queryOverride makes --stream work. The generated command decodes the whole -// response as a single JSON object, so a streamed (server-sent events) response -// always failed with "failed to unmarshal response body". Non-streaming queries -// still run the generated code unchanged. +// queryOverride sends the query itself instead of through the SDK's typed +// Query call, for two reasons: +// - The SDK decodes the whole response as one JSON object, so a streamed +// (server-sent events) response always failed with "failed to unmarshal +// response body". Streamed chunks are now printed one per line. +// - The SDK's response type only has some of the fields endpoints return, +// and drops the rest without a trace (for example choices[].finish_reason, +// tool calls, or an agent's custom outputs). The output is still the typed +// response, so it doesn't change, but a warning on stderr now names the +// fields it left out. func queryOverride(queryCmd *cobra.Command, queryReq *serving.QueryEndpointInput) { queryCmd.Long += queryStreamHelp - generatedRunE := queryCmd.RunE queryCmd.RunE = func(cmd *cobra.Command, args []string) error { - stream, err := queryWantsStream(cmd) - if err != nil || !stream { - return generatedRunE(cmd, args) - } - ctx := cmd.Context() - if cmd.Flags().Changed("json") { - jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag) - if !ok { - return generatedRunE(cmd, args) - } - diags := jsonFlag.Unmarshal(queryReq) - if diags.HasError() { - return diags.Error() - } - if len(diags) > 0 { - if err := cmdio.RenderDiagnostics(ctx, diags); err != nil { - return err - } - } + if err := unmarshalQueryJSON(cmd, queryReq); err != nil { + return err } queryReq.Name = args[0] - queryReq.Stream = true - w := cmdctx.WorkspaceClient(ctx) - apiClient, err := client.New(w.Config) + accept := "application/json" + if queryReq.Stream { + accept = "text/event-stream" + } + body, err := postInvocations(ctx, queryReq, accept) if err != nil { return err } - headers := map[string]string{ - "Accept": "text/event-stream", - "Content-Type": "application/json", + defer body.Close() + if queryReq.Stream { + return copyEventStream(cmd.OutOrStdout(), body) } - maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config)) - var body io.ReadCloser - path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name) - err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body) + + raw, err := io.ReadAll(body) if err != nil { return err } - defer body.Close() - return copyEventStream(cmd.OutOrStdout(), body) + var response serving.QueryEndpointResponse + if err := json.Unmarshal(raw, &response); err != nil { + return fmt.Errorf("failed to unmarshal response body: %w", err) + } + if err := warnDroppedFields(ctx, queryReq.Name, raw, &response); err != nil { + return err + } + return cmdio.Render(ctx, &response) } } -// queryWantsStream reports whether --stream is set, or --json sets "stream": true. -func queryWantsStream(cmd *cobra.Command) (bool, error) { - if stream, err := cmd.Flags().GetBool("stream"); err != nil || stream { - return stream, err - } +// unmarshalQueryJSON applies --json the same way the generated command does. +func unmarshalQueryJSON(cmd *cobra.Command, queryReq *serving.QueryEndpointInput) error { if !cmd.Flags().Changed("json") { - return false, nil + return nil } jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag) if !ok { - return false, nil + return errors.New("unexpected type for --json flag") + } + diags := jsonFlag.Unmarshal(queryReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + return cmdio.RenderDiagnostics(cmd.Context(), diags) + } + return nil +} + +// postInvocations sends the query with the same path, headers and body as the +// SDK's Query call, and returns the response body unread. +func postInvocations(ctx context.Context, queryReq *serving.QueryEndpointInput, accept string) (io.ReadCloser, error) { + w := cmdctx.WorkspaceClient(ctx) + apiClient, err := client.New(w.Config) + if err != nil { + return nil, err } - var body struct { - Stream bool `json:"stream"` + headers := map[string]string{ + "Accept": accept, + "Content-Type": "application/json", } - // Invalid JSON is reported by the generated command, so ignore it here. - _ = json.Unmarshal(jsonFlag.Raw(), &body) - return body.Stream, nil + maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config)) + var body io.ReadCloser + path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name) + err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body) + return body, err } // copyEventStream writes the data payload of each server-sent event as one diff --git a/cmd/workspace/serving-endpoints/query_fields.go b/cmd/workspace/serving-endpoints/query_fields.go new file mode 100644 index 00000000000..ee5c22c0f50 --- /dev/null +++ b/cmd/workspace/serving-endpoints/query_fields.go @@ -0,0 +1,148 @@ +package serving_endpoints + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" +) + +// maxDroppedFields caps how many dropped fields the warning lists. +const maxDroppedFields = 10 + +// warnDroppedFields prints a warning on stderr when the typed response that +// query prints is missing fields of the raw response. +func warnDroppedFields(ctx context.Context, name string, raw []byte, typed any) error { + dropped := droppedFields(raw, typed) + if len(dropped) == 0 { + return nil + } + list := strings.Join(dropped[:min(len(dropped), maxDroppedFields)], ", ") + if len(dropped) > maxDroppedFields { + list += fmt.Sprintf(", and %d more", len(dropped)-maxDroppedFields) + } + return cmdio.RenderDiagnostics(ctx, diag.Diagnostics{{ + Severity: diag.Warning, + Summary: "the endpoint's response has fields that this output doesn't show: " + list, + Detail: fmt.Sprintf("To see the full response, send the same request with:\n databricks api post /serving-endpoints/%s/invocations --json ...", name), + }}) +} + +// droppedFields lists the paths (like "choices[].finish_reason") that are in +// raw but are missing or have a different JSON type in typed once it's +// marshaled back to JSON. Null and zero values are ignored, because typed +// fields with omitempty leave them out. It returns nil if either side can't be +// decoded: the response already decoded once, and a failed comparison must +// not fail the query. +func droppedFields(raw []byte, typed any) []string { + rawValue, err := decodeJSON(raw) + if err != nil { + return nil + } + typedJSON, err := json.Marshal(typed) + if err != nil { + return nil + } + typedValue, err := decodeJSON(typedJSON) + if err != nil { + return nil + } + var paths []string + collectDropped(rawValue, typedValue, "", &paths) + slices.Sort(paths) + return slices.Compact(paths) +} + +func decodeJSON(data []byte) (any, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var v any + err := dec.Decode(&v) + return v, err +} + +func collectDropped(raw, typed any, path string, paths *[]string) { + switch r := raw.(type) { + case map[string]any: + t, ok := typed.(map[string]any) + if !ok { + *paths = append(*paths, path) + return + } + for k, rv := range r { + child := joinFieldPath(path, k) + tv, ok := t[k] + if !ok { + if !isZeroJSON(rv) { + *paths = append(*paths, child) + } + continue + } + collectDropped(rv, tv, child, paths) + } + case []any: + t, ok := typed.([]any) + if !ok { + *paths = append(*paths, path) + return + } + for i, rv := range r { + if i >= len(t) { + *paths = append(*paths, path+"[]") + return + } + collectDropped(rv, t[i], path+"[]", paths) + } + default: + if raw != nil && typed != nil && jsonKind(raw) != jsonKind(typed) { + *paths = append(*paths, path) + } + } +} + +func joinFieldPath(path, key string) string { + if path == "" { + return key + } + return path + "." + key +} + +func jsonKind(v any) string { + switch v.(type) { + case map[string]any: + return "object" + case []any: + return "array" + case string: + return "string" + case json.Number: + return "number" + case bool: + return "bool" + } + return "null" +} + +func isZeroJSON(v any) bool { + switch x := v.(type) { + case nil: + return true + case string: + return x == "" + case bool: + return !x + case json.Number: + f, err := x.Float64() + return err == nil && f == 0 + case []any: + return len(x) == 0 + case map[string]any: + return len(x) == 0 + } + return false +} diff --git a/cmd/workspace/serving-endpoints/query_fields_test.go b/cmd/workspace/serving-endpoints/query_fields_test.go new file mode 100644 index 00000000000..2903be5b0c8 --- /dev/null +++ b/cmd/workspace/serving-endpoints/query_fields_test.go @@ -0,0 +1,53 @@ +package serving_endpoints + +import ( + "encoding/json" + "testing" + + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDroppedFields(t *testing.T) { + tests := []struct { + name string + raw string + want []string + }{ + { + name: "nothing dropped", + raw: `{"predictions": [1, 2], "id": "x"}`, + }, + { + name: "chat fields the typed response doesn't have", + raw: `{"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi", "tool_calls": [{"id": "1"}]}}], "service_tier": "default"}`, + want: []string{"choices[].finish_reason", "choices[].message.tool_calls", "service_tier"}, + }, + { + name: "content parts become a string", + raw: `{"choices": [{"message": {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}}]}`, + want: []string{"choices[].message.content"}, + }, + { + name: "null and zero values are not reported", + raw: `{"id": null, "created": 0, "predictions": [1], "custom_outputs": {}}`, + }, + { + name: "agent output", + raw: `{"output": [{"type": "message"}], "custom_outputs": {"k": 1}}`, + want: []string{"custom_outputs", "output"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var typed serving.QueryEndpointResponse + require.NoError(t, json.Unmarshal([]byte(tt.raw), &typed)) + assert.Equal(t, tt.want, droppedFields([]byte(tt.raw), &typed)) + }) + } +} + +func TestDroppedFieldsInvalidJSON(t *testing.T) { + assert.Nil(t, droppedFields([]byte(`{"predictions": [`), &serving.QueryEndpointResponse{})) +} diff --git a/cmd/workspace/serving-endpoints/query_test.go b/cmd/workspace/serving-endpoints/query_test.go index f8b0034e74e..8404570ce3d 100644 --- a/cmd/workspace/serving-endpoints/query_test.go +++ b/cmd/workspace/serving-endpoints/query_test.go @@ -5,8 +5,6 @@ import ( "strings" "testing" - "github.com/databricks/cli/libs/flags" - "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -53,37 +51,3 @@ func TestCopyEventStreamLongLine(t *testing.T) { require.NoError(t, copyEventStream(&out, strings.NewReader("data: \""+long+"\"\n"))) assert.Equal(t, "\""+long+"\"\n", out.String()) } - -func queryCmdWithFlags(t *testing.T, args ...string) *cobra.Command { - t.Helper() - cmd := &cobra.Command{} - var jf flags.JsonFlag - var stream bool - cmd.Flags().Var(&jf, "json", "") - cmd.Flags().BoolVar(&stream, "stream", false, "") - require.NoError(t, cmd.ParseFlags(args)) - return cmd -} - -func TestQueryWantsStream(t *testing.T) { - tests := []struct { - name string - args []string - want bool - }{ - {"no flags", nil, false}, - {"stream flag", []string{"--stream"}, true}, - {"stream false flag", []string{"--stream=false"}, false}, - {"json stream true", []string{"--json", `{"stream": true}`}, true}, - {"json stream false", []string{"--json", `{"stream": false}`}, false}, - {"json without stream", []string{"--json", `{"inputs": [1]}`}, false}, - {"invalid json", []string{"--json", `{"stream": `}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := queryWantsStream(queryCmdWithFlags(t, tt.args...)) - require.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } -}