Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `serving-endpoints query` now prints a warning on stderr that names the response fields its output leaves out (for example `choices[].finish_reason`), and how to see the full response with `databricks api post`. The output itself is unchanged. ([#7664](https://github.com/databricks/cli/pull/7664))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@

=== Fields the output doesn't show are named on stderr; stdout is unchanged

>>> [CLI] serving-endpoints query chat-endpoint --json {"messages": [{"role": "user", "content": "hi"}]}
Warning: the endpoint's response has fields that this output doesn't show: choices[].finish_reason, service_tier

To see the full response, send the same request with:
databricks api post /serving-endpoints/chat-endpoint/invocations --json ...

{
"choices": [
{
"index": 0,
"message": {
"content": "Hello",
"role": "assistant"
}
}
],
"id": "chatcmpl-1",
"object": "chat.completion"
}

=== No warning when nothing is dropped

>>> [CLI] serving-endpoints query custom-endpoint --json {"dataframe_records": [{"text": "I love this"}]}
{
"predictions": [
{
"confidence": 0.9,
"label": "positive"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
title "Fields the output doesn't show are named on stderr; stdout is unchanged"
trace $CLI serving-endpoints query chat-endpoint --json '{"messages": [{"role": "user", "content": "hi"}]}'

title "No warning when nothing is dropped"
trace $CLI serving-endpoints query custom-endpoint --json '{"dataframe_records": [{"text": "I love this"}]}'
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
Cloud = false

# The testserver doesn't model invocations; stub a chat response and a custom
# model response.
[[Server]]
Pattern = "POST /serving-endpoints/chat-endpoint/invocations"
Response.Body = '''
{"id": "chatcmpl-1", "object": "chat.completion", "service_tier": "default", "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Hello"}}]}
'''

[[Server]]
Pattern = "POST /serving-endpoints/custom-endpoint/invocations"
Response.Body = '''
{"predictions": [{"label": "positive", "confidence": 0.9}]}
'''
110 changes: 62 additions & 48 deletions cmd/workspace/serving-endpoints/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package serving_endpoints
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
Expand All @@ -30,77 +32,89 @@ const queryStreamHelp = `
For a chat endpoint, the reply text of each chunk is in
.choices[0].delta.content.`

// queryOverride makes --stream work. The generated command decodes the whole
// response as a single JSON object, so a streamed (server-sent events) response
// always failed with "failed to unmarshal response body". Non-streaming queries
// still run the generated code unchanged.
// queryOverride sends the query itself instead of through the SDK's typed
// Query call, for two reasons:
// - The SDK decodes the whole response as one JSON object, so a streamed
// (server-sent events) response always failed with "failed to unmarshal
// response body". Streamed chunks are now printed one per line.
// - The SDK's response type only has some of the fields endpoints return,
// and drops the rest without a trace (for example choices[].finish_reason,
// tool calls, or an agent's custom outputs). The output is still the typed
// response, so it doesn't change, but a warning on stderr now names the
// fields it left out.
func queryOverride(queryCmd *cobra.Command, queryReq *serving.QueryEndpointInput) {
queryCmd.Long += queryStreamHelp
generatedRunE := queryCmd.RunE
queryCmd.RunE = func(cmd *cobra.Command, args []string) error {
stream, err := queryWantsStream(cmd)
if err != nil || !stream {
return generatedRunE(cmd, args)
}

ctx := cmd.Context()
if cmd.Flags().Changed("json") {
jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag)
if !ok {
return generatedRunE(cmd, args)
}
diags := jsonFlag.Unmarshal(queryReq)
if diags.HasError() {
return diags.Error()
}
if len(diags) > 0 {
if err := cmdio.RenderDiagnostics(ctx, diags); err != nil {
return err
}
}
if err := unmarshalQueryJSON(cmd, queryReq); err != nil {
return err
}
queryReq.Name = args[0]
queryReq.Stream = true

w := cmdctx.WorkspaceClient(ctx)
apiClient, err := client.New(w.Config)
accept := "application/json"
if queryReq.Stream {
accept = "text/event-stream"
}
body, err := postInvocations(ctx, queryReq, accept)
if err != nil {
return err
}
headers := map[string]string{
"Accept": "text/event-stream",
"Content-Type": "application/json",
defer body.Close()
if queryReq.Stream {
return copyEventStream(cmd.OutOrStdout(), body)
}
maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config))
var body io.ReadCloser
path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name)
err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body)

raw, err := io.ReadAll(body)
if err != nil {
return err
}
defer body.Close()
return copyEventStream(cmd.OutOrStdout(), body)
var response serving.QueryEndpointResponse
if err := json.Unmarshal(raw, &response); err != nil {
return fmt.Errorf("failed to unmarshal response body: %w", err)
}
if err := warnDroppedFields(ctx, queryReq.Name, raw, &response); err != nil {
return err
}
return cmdio.Render(ctx, &response)
}
}

// queryWantsStream reports whether --stream is set, or --json sets "stream": true.
func queryWantsStream(cmd *cobra.Command) (bool, error) {
if stream, err := cmd.Flags().GetBool("stream"); err != nil || stream {
return stream, err
}
// unmarshalQueryJSON applies --json the same way the generated command does.
func unmarshalQueryJSON(cmd *cobra.Command, queryReq *serving.QueryEndpointInput) error {
if !cmd.Flags().Changed("json") {
return false, nil
return nil
}
jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag)
if !ok {
return false, nil
return errors.New("unexpected type for --json flag")
}
diags := jsonFlag.Unmarshal(queryReq)
if diags.HasError() {
return diags.Error()
}
if len(diags) > 0 {
return cmdio.RenderDiagnostics(cmd.Context(), diags)
}
return nil
}

// postInvocations sends the query with the same path, headers and body as the
// SDK's Query call, and returns the response body unread.
func postInvocations(ctx context.Context, queryReq *serving.QueryEndpointInput, accept string) (io.ReadCloser, error) {
w := cmdctx.WorkspaceClient(ctx)
apiClient, err := client.New(w.Config)
if err != nil {
return nil, err
}
var body struct {
Stream bool `json:"stream"`
headers := map[string]string{
"Accept": accept,
"Content-Type": "application/json",
}
// Invalid JSON is reported by the generated command, so ignore it here.
_ = json.Unmarshal(jsonFlag.Raw(), &body)
return body.Stream, nil
maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config))
var body io.ReadCloser
path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name)
err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body)
return body, err
}

// copyEventStream writes the data payload of each server-sent event as one
Expand Down
148 changes: 148 additions & 0 deletions cmd/workspace/serving-endpoints/query_fields.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
package serving_endpoints

import (
"bytes"
"context"
"encoding/json"
"fmt"
"slices"
"strings"

"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
)

// maxDroppedFields caps how many dropped fields the warning lists.
const maxDroppedFields = 10

// warnDroppedFields prints a warning on stderr when the typed response that
// query prints is missing fields of the raw response.
func warnDroppedFields(ctx context.Context, name string, raw []byte, typed any) error {
dropped := droppedFields(raw, typed)
if len(dropped) == 0 {
return nil
}
list := strings.Join(dropped[:min(len(dropped), maxDroppedFields)], ", ")
if len(dropped) > maxDroppedFields {
list += fmt.Sprintf(", and %d more", len(dropped)-maxDroppedFields)
}
return cmdio.RenderDiagnostics(ctx, diag.Diagnostics{{
Severity: diag.Warning,
Summary: "the endpoint's response has fields that this output doesn't show: " + list,
Detail: fmt.Sprintf("To see the full response, send the same request with:\n databricks api post /serving-endpoints/%s/invocations --json ...", name),
}})
}

// droppedFields lists the paths (like "choices[].finish_reason") that are in
// raw but are missing or have a different JSON type in typed once it's
// marshaled back to JSON. Null and zero values are ignored, because typed
// fields with omitempty leave them out. It returns nil if either side can't be
// decoded: the response already decoded once, and a failed comparison must
// not fail the query.
func droppedFields(raw []byte, typed any) []string {
rawValue, err := decodeJSON(raw)
if err != nil {
return nil
}
typedJSON, err := json.Marshal(typed)
if err != nil {
return nil
}
typedValue, err := decodeJSON(typedJSON)
if err != nil {
return nil
}
var paths []string
collectDropped(rawValue, typedValue, "", &paths)
slices.Sort(paths)
return slices.Compact(paths)
}

func decodeJSON(data []byte) (any, error) {
dec := json.NewDecoder(bytes.NewReader(data))
dec.UseNumber()
var v any
err := dec.Decode(&v)
return v, err
}

func collectDropped(raw, typed any, path string, paths *[]string) {
switch r := raw.(type) {
case map[string]any:
t, ok := typed.(map[string]any)
if !ok {
*paths = append(*paths, path)
return
}
for k, rv := range r {
child := joinFieldPath(path, k)
tv, ok := t[k]
if !ok {
if !isZeroJSON(rv) {
*paths = append(*paths, child)
}
continue
}
collectDropped(rv, tv, child, paths)
}
case []any:
t, ok := typed.([]any)
if !ok {
*paths = append(*paths, path)
return
}
for i, rv := range r {
if i >= len(t) {
*paths = append(*paths, path+"[]")
return
}
collectDropped(rv, t[i], path+"[]", paths)
}
default:
if raw != nil && typed != nil && jsonKind(raw) != jsonKind(typed) {
*paths = append(*paths, path)
}
}
}

func joinFieldPath(path, key string) string {
if path == "" {
return key
}
return path + "." + key
}

func jsonKind(v any) string {
switch v.(type) {
case map[string]any:
return "object"
case []any:
return "array"
case string:
return "string"
case json.Number:
return "number"
case bool:
return "bool"
}
return "null"
}

func isZeroJSON(v any) bool {
switch x := v.(type) {
case nil:
return true
case string:
return x == ""
case bool:
return !x
case json.Number:
f, err := x.Float64()
return err == nil && f == 0
case []any:
return len(x) == 0
case map[string]any:
return len(x) == 0
}
return false
}
Loading
Loading