Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/serving-endpoints-query-stream.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `serving-endpoints query --stream` (or `"stream": true` in `--json`) now prints the streamed response as it arrives, one JSON chunk per line. It previously always failed with `failed to unmarshal response body`. ([#7660](https://github.com/databricks/cli/pull/7660))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@

=== --stream prints one JSON chunk per line

>>> [CLI] serving-endpoints query chat-endpoint --stream --json {"messages": [{"role": "user", "content": "hi"}]}
{"choices":[{"delta":{"content":"Hel"},"index":0}]}
{"choices":[{"delta":{"content":"lo"},"finish_reason":"stop","index":0}]}

=== "stream": true in --json also streams

>>> [CLI] serving-endpoints query chat-endpoint --json {"messages": [{"role": "user", "content": "hi"}], "stream": true}
Hello

>>> print_requests.py //serving-endpoints
{
"method": "POST",
"path": "/serving-endpoints/chat-endpoint/invocations",
"body": {
"messages": [
{
"content": "hi",
"role": "user"
}
],
"stream": true
}
}
{
"method": "POST",
"path": "/serving-endpoints/chat-endpoint/invocations",
"body": {
"messages": [
{
"content": "hi",
"role": "user"
}
],
"stream": true
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
title "--stream prints one JSON chunk per line"
trace $CLI serving-endpoints query chat-endpoint --stream --json '{"messages": [{"role": "user", "content": "hi"}]}'

title "\"stream\": true in --json also streams"
trace $CLI serving-endpoints query chat-endpoint --json '{"messages": [{"role": "user", "content": "hi"}], "stream": true}' | jq -rj '.choices[0].delta.content'
echo

trace print_requests.py //serving-endpoints
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
Cloud = false
RecordRequests = true

# The testserver doesn't model invocations; stub a streamed chat response.
[[Server]]
Pattern = "POST /serving-endpoints/chat-endpoint/invocations"
Response.Body = '''
data: {"choices":[{"delta":{"content":"Hel"},"index":0}]}

data: {"choices":[{"delta":{"content":"lo"},"finish_reason":"stop","index":0}]}

data: [DONE]

'''
[Server.Response.Headers]
Content-Type = ["text/event-stream"]
147 changes: 147 additions & 0 deletions cmd/workspace/serving-endpoints/query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package serving_endpoints

import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"maps"
"net/http"

"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/databricks-sdk-go/client"
"github.com/databricks/databricks-sdk-go/service/serving"
"github.com/spf13/cobra"
)

const (
sseDataPrefix = "data:"
sseDone = "[DONE]"
)

const queryStreamHelp = `

With --stream (or "stream": true in --json), the response is printed as it
arrives: one JSON chunk per line, without the server-sent event framing.
For a chat endpoint, the reply text of each chunk is in
.choices[0].delta.content.`

// queryOverride makes --stream work. The generated command decodes the whole
// response as a single JSON object, so a streamed (server-sent events) response
// always failed with "failed to unmarshal response body". Non-streaming queries
// still run the generated code unchanged.
func queryOverride(queryCmd *cobra.Command, queryReq *serving.QueryEndpointInput) {
queryCmd.Long += queryStreamHelp
generatedRunE := queryCmd.RunE
queryCmd.RunE = func(cmd *cobra.Command, args []string) error {
stream, err := queryWantsStream(cmd)
if err != nil || !stream {
return generatedRunE(cmd, args)
}

ctx := cmd.Context()
if cmd.Flags().Changed("json") {
jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag)
if !ok {
return generatedRunE(cmd, args)
}
diags := jsonFlag.Unmarshal(queryReq)
if diags.HasError() {
return diags.Error()
}
if len(diags) > 0 {
if err := cmdio.RenderDiagnostics(ctx, diags); err != nil {
return err
}
}
}
queryReq.Name = args[0]
queryReq.Stream = true

w := cmdctx.WorkspaceClient(ctx)
apiClient, err := client.New(w.Config)
if err != nil {
return err
}
headers := map[string]string{
"Accept": "text/event-stream",
"Content-Type": "application/json",
}
maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config))
var body io.ReadCloser
path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name)
err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body)
if err != nil {
return err
}
defer body.Close()
return copyEventStream(cmd.OutOrStdout(), body)
}
}

// queryWantsStream reports whether --stream is set, or --json sets "stream": true.
func queryWantsStream(cmd *cobra.Command) (bool, error) {
if stream, err := cmd.Flags().GetBool("stream"); err != nil || stream {
return stream, err
}
if !cmd.Flags().Changed("json") {
return false, nil
}
jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag)
if !ok {
return false, nil
}
var body struct {
Stream bool `json:"stream"`
}
// Invalid JSON is reported by the generated command, so ignore it here.
_ = json.Unmarshal(jsonFlag.Raw(), &body)
return body.Stream, nil
}

// copyEventStream writes the data payload of each server-sent event as one
// line, and stops at the "[DONE]" sentinel. Lines that aren't SSE framing (for
// example a plain JSON body from a model that ignores "stream") are copied as is.
func copyEventStream(out io.Writer, body io.Reader) error {
scanner := bufio.NewScanner(body)
// Chunks can be large (for example tool call arguments), so allow long lines.
scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
for scanner.Scan() {
line := scanner.Bytes()
if payload, ok := bytes.CutPrefix(line, []byte(sseDataPrefix)); ok {
payload = bytes.TrimSpace(payload)
if string(payload) == sseDone {
return nil
}
line = payload
} else if isSSEFraming(line) {
continue
}
if _, err := fmt.Fprintf(out, "%s\n", line); err != nil {
return err
}
}
return scanner.Err()
}

// isSSEFraming reports whether line is an SSE blank line, comment or non-data field.
func isSSEFraming(line []byte) bool {
line = bytes.TrimSpace(line)
if len(line) == 0 || line[0] == ':' {
return true
}
for _, field := range []string{"event:", "id:", "retry:"} {
if bytes.HasPrefix(line, []byte(field)) {
return true
}
}
return false
}

func init() {
queryOverrides = append(queryOverrides, queryOverride)
}
89 changes: 89 additions & 0 deletions cmd/workspace/serving-endpoints/query_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package serving_endpoints

import (
"bytes"
"strings"
"testing"

"github.com/databricks/cli/libs/flags"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestCopyEventStream(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{
name: "data lines until done",
in: "data: {\"a\":1}\n\ndata:{\"a\":2}\n\ndata: [DONE]\n\ndata: {\"after\":true}\n",
want: "{\"a\":1}\n{\"a\":2}\n",
},
{
name: "skips comments and other SSE fields",
in: ": keep-alive\nevent: message\nid: 7\nretry: 1000\ndata: {\"a\":1}\n\n",
want: "{\"a\":1}\n",
},
{
name: "plain JSON body is copied as is",
in: "{\"predictions\":[1]}\n",
want: "{\"predictions\":[1]}\n",
},
{
name: "stream without done sentinel",
in: "data: {\"a\":1}\n",
want: "{\"a\":1}\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var out bytes.Buffer
require.NoError(t, copyEventStream(&out, strings.NewReader(tt.in)))
assert.Equal(t, tt.want, out.String())
})
}
}

func TestCopyEventStreamLongLine(t *testing.T) {
long := strings.Repeat("x", 1024*1024)
var out bytes.Buffer
require.NoError(t, copyEventStream(&out, strings.NewReader("data: \""+long+"\"\n")))
assert.Equal(t, "\""+long+"\"\n", out.String())
}

func queryCmdWithFlags(t *testing.T, args ...string) *cobra.Command {
t.Helper()
cmd := &cobra.Command{}
var jf flags.JsonFlag
var stream bool
cmd.Flags().Var(&jf, "json", "")
cmd.Flags().BoolVar(&stream, "stream", false, "")
require.NoError(t, cmd.ParseFlags(args))
return cmd
}

func TestQueryWantsStream(t *testing.T) {
tests := []struct {
name string
args []string
want bool
}{
{"no flags", nil, false},
{"stream flag", []string{"--stream"}, true},
{"stream false flag", []string{"--stream=false"}, false},
{"json stream true", []string{"--json", `{"stream": true}`}, true},
{"json stream false", []string{"--json", `{"stream": false}`}, false},
{"json without stream", []string{"--json", `{"inputs": [1]}`}, false},
{"invalid json", []string{"--json", `{"stream": `}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := queryWantsStream(queryCmdWithFlags(t, tt.args...))
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}
Loading