diff --git a/.nextchanges/cli/serving-endpoints-query-stream.md b/.nextchanges/cli/serving-endpoints-query-stream.md new file mode 100644 index 00000000000..d42abd27c31 --- /dev/null +++ b/.nextchanges/cli/serving-endpoints-query-stream.md @@ -0,0 +1 @@ +* `serving-endpoints query --stream` (or `"stream": true` in `--json`) now prints the streamed response as it arrives, one JSON chunk per line. It previously always failed with `failed to unmarshal response body`. ([#7660](https://github.com/databricks/cli/pull/7660)) diff --git a/acceptance/cmd/workspace/serving-endpoints/query-stream/out.test.toml b/acceptance/cmd/workspace/serving-endpoints/query-stream/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-stream/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/workspace/serving-endpoints/query-stream/output.txt b/acceptance/cmd/workspace/serving-endpoints/query-stream/output.txt new file mode 100644 index 00000000000..896e13b9dca --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-stream/output.txt @@ -0,0 +1,39 @@ + +=== --stream prints one JSON chunk per line + +>>> [CLI] serving-endpoints query chat-endpoint --stream --json {"messages": [{"role": "user", "content": "hi"}]} +{"choices":[{"delta":{"content":"Hel"},"index":0}]} +{"choices":[{"delta":{"content":"lo"},"finish_reason":"stop","index":0}]} + +=== "stream": true in --json also streams + +>>> [CLI] serving-endpoints query chat-endpoint --json {"messages": [{"role": "user", "content": "hi"}], "stream": true} +Hello + +>>> print_requests.py //serving-endpoints +{ + "method": "POST", + "path": "/serving-endpoints/chat-endpoint/invocations", + "body": { + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "stream": true + } +} +{ + "method": "POST", + "path": "/serving-endpoints/chat-endpoint/invocations", + "body": { + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "stream": true + } +} diff --git a/acceptance/cmd/workspace/serving-endpoints/query-stream/script b/acceptance/cmd/workspace/serving-endpoints/query-stream/script new file mode 100644 index 00000000000..0a65baecf3e --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-stream/script @@ -0,0 +1,8 @@ +title "--stream prints one JSON chunk per line" +trace $CLI serving-endpoints query chat-endpoint --stream --json '{"messages": [{"role": "user", "content": "hi"}]}' + +title "\"stream\": true in --json also streams" +trace $CLI serving-endpoints query chat-endpoint --json '{"messages": [{"role": "user", "content": "hi"}], "stream": true}' | jq -rj '.choices[0].delta.content' +echo + +trace print_requests.py //serving-endpoints diff --git a/acceptance/cmd/workspace/serving-endpoints/query-stream/test.toml b/acceptance/cmd/workspace/serving-endpoints/query-stream/test.toml new file mode 100644 index 00000000000..e878fa78665 --- /dev/null +++ b/acceptance/cmd/workspace/serving-endpoints/query-stream/test.toml @@ -0,0 +1,16 @@ +Cloud = false +RecordRequests = true + +# The testserver doesn't model invocations; stub a streamed chat response. +[[Server]] +Pattern = "POST /serving-endpoints/chat-endpoint/invocations" +Response.Body = ''' +data: {"choices":[{"delta":{"content":"Hel"},"index":0}]} + +data: {"choices":[{"delta":{"content":"lo"},"finish_reason":"stop","index":0}]} + +data: [DONE] + +''' +[Server.Response.Headers] +Content-Type = ["text/event-stream"] diff --git a/cmd/workspace/serving-endpoints/query.go b/cmd/workspace/serving-endpoints/query.go new file mode 100644 index 00000000000..551a674fe08 --- /dev/null +++ b/cmd/workspace/serving-endpoints/query.go @@ -0,0 +1,147 @@ +package serving_endpoints + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/spf13/cobra" +) + +const ( + sseDataPrefix = "data:" + sseDone = "[DONE]" +) + +const queryStreamHelp = ` + + With --stream (or "stream": true in --json), the response is printed as it + arrives: one JSON chunk per line, without the server-sent event framing. + For a chat endpoint, the reply text of each chunk is in + .choices[0].delta.content.` + +// queryOverride makes --stream work. The generated command decodes the whole +// response as a single JSON object, so a streamed (server-sent events) response +// always failed with "failed to unmarshal response body". Non-streaming queries +// still run the generated code unchanged. +func queryOverride(queryCmd *cobra.Command, queryReq *serving.QueryEndpointInput) { + queryCmd.Long += queryStreamHelp + generatedRunE := queryCmd.RunE + queryCmd.RunE = func(cmd *cobra.Command, args []string) error { + stream, err := queryWantsStream(cmd) + if err != nil || !stream { + return generatedRunE(cmd, args) + } + + ctx := cmd.Context() + if cmd.Flags().Changed("json") { + jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag) + if !ok { + return generatedRunE(cmd, args) + } + diags := jsonFlag.Unmarshal(queryReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + if err := cmdio.RenderDiagnostics(ctx, diags); err != nil { + return err + } + } + } + queryReq.Name = args[0] + queryReq.Stream = true + + w := cmdctx.WorkspaceClient(ctx) + apiClient, err := client.New(w.Config) + if err != nil { + return err + } + headers := map[string]string{ + "Accept": "text/event-stream", + "Content-Type": "application/json", + } + maps.Copy(headers, auth.WorkspaceIDHeaders(w.Config)) + var body io.ReadCloser + path := fmt.Sprintf("/serving-endpoints/%v/invocations", queryReq.Name) + err = apiClient.Do(ctx, http.MethodPost, path, headers, nil, queryReq, &body) + if err != nil { + return err + } + defer body.Close() + return copyEventStream(cmd.OutOrStdout(), body) + } +} + +// queryWantsStream reports whether --stream is set, or --json sets "stream": true. +func queryWantsStream(cmd *cobra.Command) (bool, error) { + if stream, err := cmd.Flags().GetBool("stream"); err != nil || stream { + return stream, err + } + if !cmd.Flags().Changed("json") { + return false, nil + } + jsonFlag, ok := cmd.Flags().Lookup("json").Value.(*flags.JsonFlag) + if !ok { + return false, nil + } + var body struct { + Stream bool `json:"stream"` + } + // Invalid JSON is reported by the generated command, so ignore it here. + _ = json.Unmarshal(jsonFlag.Raw(), &body) + return body.Stream, nil +} + +// copyEventStream writes the data payload of each server-sent event as one +// line, and stops at the "[DONE]" sentinel. Lines that aren't SSE framing (for +// example a plain JSON body from a model that ignores "stream") are copied as is. +func copyEventStream(out io.Writer, body io.Reader) error { + scanner := bufio.NewScanner(body) + // Chunks can be large (for example tool call arguments), so allow long lines. + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for scanner.Scan() { + line := scanner.Bytes() + if payload, ok := bytes.CutPrefix(line, []byte(sseDataPrefix)); ok { + payload = bytes.TrimSpace(payload) + if string(payload) == sseDone { + return nil + } + line = payload + } else if isSSEFraming(line) { + continue + } + if _, err := fmt.Fprintf(out, "%s\n", line); err != nil { + return err + } + } + return scanner.Err() +} + +// isSSEFraming reports whether line is an SSE blank line, comment or non-data field. +func isSSEFraming(line []byte) bool { + line = bytes.TrimSpace(line) + if len(line) == 0 || line[0] == ':' { + return true + } + for _, field := range []string{"event:", "id:", "retry:"} { + if bytes.HasPrefix(line, []byte(field)) { + return true + } + } + return false +} + +func init() { + queryOverrides = append(queryOverrides, queryOverride) +} diff --git a/cmd/workspace/serving-endpoints/query_test.go b/cmd/workspace/serving-endpoints/query_test.go new file mode 100644 index 00000000000..f8b0034e74e --- /dev/null +++ b/cmd/workspace/serving-endpoints/query_test.go @@ -0,0 +1,89 @@ +package serving_endpoints + +import ( + "bytes" + "strings" + "testing" + + "github.com/databricks/cli/libs/flags" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyEventStream(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "data lines until done", + in: "data: {\"a\":1}\n\ndata:{\"a\":2}\n\ndata: [DONE]\n\ndata: {\"after\":true}\n", + want: "{\"a\":1}\n{\"a\":2}\n", + }, + { + name: "skips comments and other SSE fields", + in: ": keep-alive\nevent: message\nid: 7\nretry: 1000\ndata: {\"a\":1}\n\n", + want: "{\"a\":1}\n", + }, + { + name: "plain JSON body is copied as is", + in: "{\"predictions\":[1]}\n", + want: "{\"predictions\":[1]}\n", + }, + { + name: "stream without done sentinel", + in: "data: {\"a\":1}\n", + want: "{\"a\":1}\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var out bytes.Buffer + require.NoError(t, copyEventStream(&out, strings.NewReader(tt.in))) + assert.Equal(t, tt.want, out.String()) + }) + } +} + +func TestCopyEventStreamLongLine(t *testing.T) { + long := strings.Repeat("x", 1024*1024) + var out bytes.Buffer + require.NoError(t, copyEventStream(&out, strings.NewReader("data: \""+long+"\"\n"))) + assert.Equal(t, "\""+long+"\"\n", out.String()) +} + +func queryCmdWithFlags(t *testing.T, args ...string) *cobra.Command { + t.Helper() + cmd := &cobra.Command{} + var jf flags.JsonFlag + var stream bool + cmd.Flags().Var(&jf, "json", "") + cmd.Flags().BoolVar(&stream, "stream", false, "") + require.NoError(t, cmd.ParseFlags(args)) + return cmd +} + +func TestQueryWantsStream(t *testing.T) { + tests := []struct { + name string + args []string + want bool + }{ + {"no flags", nil, false}, + {"stream flag", []string{"--stream"}, true}, + {"stream false flag", []string{"--stream=false"}, false}, + {"json stream true", []string{"--json", `{"stream": true}`}, true}, + {"json stream false", []string{"--json", `{"stream": false}`}, false}, + {"json without stream", []string{"--json", `{"inputs": [1]}`}, false}, + {"invalid json", []string{"--json", `{"stream": `}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := queryWantsStream(queryCmdWithFlags(t, tt.args...)) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +}