squash bot discovered issues

CONTRIBUTING.md

@@ -21,6 +21,33 @@ Because the SDK is new and under active development, third-party contribution be
 
 See [AGENTS.md](./AGENTS.md) for best practices developing, testing, and releasing the SDK.
 
+### Cross-language instrumentation specs
+
+Canonical span input/output and token metrics belong in
+[braintrust-spec](https://github.com/braintrustdata/braintrust-spec), not duplicated
+provider unit tests. Keep Java-only adapter boundaries and deterministic edge cases
+(such as absent usage fields or integer overflow) in unit tests.
+
+BTX fetches the revision pinned by `braintrustSpecRef` in `gradle.properties`.
+To develop against a separate spec worktree without changing that pin:
+
+```bash
+export BTX_SPEC_ROOT=/path/to/braintrust-spec/test/llm_span
+VCR_MODE=off ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
+VCR_MODE=record ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
+VCR_MODE=replay ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
+```
+
+The Responses cases exercise string input and instructions through the direct
+OpenAI SDK, plus equivalent chat messages through LangChain4j. The Google thinking
+case checks positive reasoning counts and inclusive completion-token accounting.
+The runner implements the shared `is_google_usage_metadata_normalized` predicate;
+it must not treat that predicate as a generic non-null assertion.
+
+Before merging, pin a remotely fetchable spec revision, unset `BTX_SPEC_ROOT`, and
+run `./gradlew :btx:test` in replay mode to verify the default consumption path.
+Record cassettes through VCR; never edit them by hand.
+
 ### CodeQL result caching
 
 CI installs the recommended CodeQL bundle before looking up cached SARIF results.

braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java

@@ -152,9 +152,18 @@ private void tagSpan(
                     if (usage.containsKey("promptTokenCount")) {
                         metrics.put("prompt_tokens", (Number) usage.get("promptTokenCount"));
                     }
-                    if (usage.containsKey("candidatesTokenCount")) {
+                    Number candidates = (Number) usage.get("candidatesTokenCount");
+                    Number thoughts = (Number) usage.get("thoughtsTokenCount");
+                    // Gemini reports candidates separately from thoughts; completion includes both.
+                    // A missing component contributes zero only when the other is reported.
+                    if (candidates != null || thoughts != null) {
                         metrics.put(
-                                "completion_tokens", (Number) usage.get("candidatesTokenCount"));
+                                "completion_tokens",
+                                (candidates == null ? 0L : candidates.longValue())
+                                        + (thoughts == null ? 0L : thoughts.longValue()));
+                    }
+                    if (thoughts != null) {
+                        metrics.put("completion_reasoning_tokens", thoughts);
                     }
                     if (usage.containsKey("totalTokenCount")) {
                         metrics.put("tokens", (Number) usage.get("totalTokenCount"));

braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java

@@ -26,6 +26,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
 /** Transport-level coverage without credentials, network access, or recorded batch jobs. */
@@ -227,6 +228,41 @@ void llmEndpointsAreTagged(String endpoint) throws Exception {
         }
     }
 
+    @ParameterizedTest
+    @CsvSource(
+            delimiter = '|',
+            value = {
+                "{\"promptTokenCount\":8,\"candidatesTokenCount\":7,\"totalTokenCount\":15}"
+                        + "|{\"prompt_tokens\":8,\"completion_tokens\":7,\"tokens\":15}",
+                "{\"candidatesTokenCount\":7,\"thoughtsTokenCount\":0}"
+                        + "|{\"completion_tokens\":7,\"completion_reasoning_tokens\":0}",
+                "{\"promptTokenCount\":8,\"thoughtsTokenCount\":12,\"totalTokenCount\":20}"
+                    + "|{\"prompt_tokens\":8,\"completion_tokens\":12,\"completion_reasoning_tokens\":12,\"tokens\":20}",
+                "{}|{}",
+                "{\"candidatesTokenCount\":null,\"thoughtsTokenCount\":null}|{}",
+                "{\"candidatesTokenCount\":2147483647,\"thoughtsTokenCount\":1}"
+                        + "|{\"completion_tokens\":2147483648,\"completion_reasoning_tokens\":1}"
+            })
+    void normalizesPartialAndBoundaryUsage(String usage, String expected) throws Exception {
+        delegate.response = "{\"candidates\":[],\"usageMetadata\":" + usage + "}";
+        try (ApiResponse response =
+                invoke("post", "models/gemini:generateContent", "{}", false, false)) {
+            response.getBody().string();
+            assertEquals(JSON.readTree(expected), JSON.readTree(attribute("braintrust.metrics")));
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"{}", "{\"usageMetadata\":null}"})
+    void missingUsageDoesNotInventMetrics(String output) throws Exception {
+        delegate.response = output;
+        try (ApiResponse response =
+                invoke("post", "models/gemini:generateContent", "{}", false, false)) {
+            response.getBody().string();
+            assertNull(attribute("braintrust.metrics"));
+        }
+    }
+
     @Test
     void asynchronousNonLlmSpansRetainParentAndEndOnlyWhenCompleted() throws Exception {
         for (boolean bytes : List.of(false, true)) {

braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java

@@ -322,8 +322,15 @@ private static void tagOpenAIRequest(
                 metadata.put("model", requestJson.get("model").asText());
             }
             putGenerationParameters(metadata, requestJson);
-            // Chat completions API uses "messages"; Responses API uses "input"
-            if (requestJson.has("messages")) {
+            // Only Responses requests normalize input and instructions into message entries.
+            if ("POST".equals(method)
+                    && !pathSegments.isEmpty()
+                    && "responses".equals(pathSegments.get(pathSegments.size() - 1))) {
+                JsonNode input = openAIResponsesInput(requestJson);
+                if (input != null) {
+                    span.setAttribute("braintrust.input_json", toJson(input));
+                }
+            } else if (requestJson.has("messages")) {
                 span.setAttribute("braintrust.input_json", toJson(requestJson.get("messages")));
             } else if (requestJson.has("input") && requestJson.get("input").isArray()) {
                 span.setAttribute("braintrust.input_json", toJson(requestJson.get("input")));
@@ -333,6 +340,29 @@ private static void tagOpenAIRequest(
         span.setAttribute("braintrust.metadata", toJson(metadata));
     }
 
+    @Nullable
+    private static JsonNode openAIResponsesInput(JsonNode requestJson) {
+        JsonNode input = requestJson.get("input");
+        JsonNode instructions = requestJson.get("instructions");
+        boolean hasInstructions = instructions != null && instructions.isTextual();
+        boolean hasTextInput = input != null && input.isTextual();
+        if (!hasInstructions && !hasTextInput) {
+            return input != null && input.isArray() ? input : null;
+        }
+
+        ArrayNode messages = BraintrustJsonMapper.get().createArrayNode();
+        if (hasInstructions) {
+            messages.addObject().put("role", "system").set("content", instructions);
+        }
+        if (hasTextInput) {
+            messages.addObject().put("role", "user").set("content", input);
+        } else if (input != null && input.isArray()) {
+            // Reuse heterogeneous entries, including tool calls/results and multimodal content.
+            messages.addAll((ArrayNode) input);
+        }
+        return messages;
+    }
+
     @SneakyThrows
     private static void tagOpenAIResponse(
             Span span, JsonNode responseJson, @Nullable Long timeToFirstTokenNanoseconds) {
@@ -1160,8 +1190,8 @@ private static JsonNode maybeParseJsonString(JsonNode value) {
     /**
      * Request fields that are conversation content, not generation parameters. Each is already
      * captured as the span's input, so copying it into metadata would duplicate a potentially large
-     * payload. {@code instructions} (the Responses API's system prompt) belongs in the span input
-     * too — tracked separately — so it is withheld here rather than landing in metadata.
+     * payload. {@code instructions} (the Responses API's system prompt) is prepended to the span
+     * input as a system message, rather than landing in metadata.
      *
      * <p>The list spans every endpoint reachable through a wrapped client, not just chat: {@code
      * prompt} carries the user's text on legacy OpenAI completions, Anthropic's legacy {@code

braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java

@@ -390,16 +390,192 @@ void clientToolUseAndPlainTextEmitNothing() {
      * braintrust.input_json} attribute (null when none was set).
      */
     private String inputJsonForRequest(String provider, String requestBody) {
+        return inputJsonForRequest(provider, List.of("v1", "messages"), "POST", requestBody);
+    }
+
+    private String inputJsonForRequest(
+            String provider, List<String> pathSegments, String method, String requestBody) {
         Span span = tracer.spanBuilder("llm").startSpan();
         InstrumentationSemConv.tagLLMSpanRequest(
-                span,
-                provider,
-                "https://api.anthropic.com",
-                List.of("v1", "messages"),
+                span, provider, "https://api.example.com", pathSegments, method, requestBody);
+        span.end();
+        var spans = exporter.getFinishedSpanItems();
+        return spans.get(spans.size() - 1).getAttributes().get(INPUT_JSON);
+    }
+
+    private String responsesInput(String requestBody) {
+        return inputJsonForRequest(
+                InstrumentationSemConv.PROVIDER_NAME_OPENAI,
+                List.of("v1", "responses"),
                 "POST",
                 requestBody);
-        span.end();
-        return exporter.getFinishedSpanItems().get(0).getAttributes().get(INPUT_JSON);
+    }
+
+    @Test
+    void openAiResponsesRecognizesDeploymentPrefixedEndpoint() {
+        String body =
+                """
+                {"instructions": "be terse", "input": "hello"}
+                """;
+        assertEquals(
+                json(
+                        """
+                        [{"role": "system", "content": "be terse"},
+                         {"role": "user", "content": "hello"}]
+                        """),
+                json(
+                        inputJsonForRequest(
+                                InstrumentationSemConv.PROVIDER_NAME_OPENAI,
+                                List.of("openai", "deployments", "my-model", "responses"),
+                                "POST",
+                                body)));
+    }
+
+    @Test
+    void openAiResponsesArrayPreservesToolAndMultimodalEntriesAfterInstructions() {
+        String body =
+                """
+                {
+                  "instructions": "be terse",
+                  "input": [
+                    {"role": "user", "content": [
+                      {"type": "input_text", "text": "describe this"},
+                      {"type": "input_image", "image_url": "https://example.com/image.png"}
+                    ]},
+                    {"type": "function_call", "id": "fc_1", "call_id": "call_1",
+                     "name": "get_weather", "arguments": "{\\"city\\":\\"SF\\"}"},
+                    {"type": "function_call_output", "call_id": "call_1", "output": "sunny"},
+                    {"type": "item_reference", "id": "msg_previous"}
+                  ]
+                }
+                """;
+        JsonNode originalInput = json(body).get("input");
+        JsonNode captured = json(responsesInput(body));
+        assertEquals(originalInput.size() + 1, captured.size());
+        assertEquals(
+                json(
+                        """
+                        {"role": "system", "content": "be terse"}
+                        """),
+                captured.get(0));
+        for (int i = 0; i < originalInput.size(); i++) {
+            assertEquals(originalInput.get(i), captured.get(i + 1));
+        }
+
+        var withoutInstructions = json(body).deepCopy();
+        ((com.fasterxml.jackson.databind.node.ObjectNode) withoutInstructions)
+                .remove("instructions");
+        assertEquals(originalInput, json(responsesInput(withoutInstructions.toString())));
+    }
+
+    @Test
+    void openAiResponsesInstructionsOnlyContinuationOmitsMissingOrNullInput() {
+        for (String body :
+                List.of(
+                        """
+                        {"instructions": "continue tersely", "previous_response_id": "resp_1"}
+                        """,
+                        """
+                        {"instructions": "continue tersely", "previous_response_id": "resp_1",
+                         "input": null}
+                        """)) {
+            assertEquals(
+                    json(
+                            """
+                            [{"role": "system", "content": "continue tersely"}]
+                            """),
+                    json(responsesInput(body)));
+        }
+    }
+
+    @Test
+    void openAiResponsesAbsentOrNullInstructionsDoNotAddSystemMessage() {
+        JsonNode expected =
+                json(
+                        """
+                [{"role": "user", "content": "hello"}]
+                """);
+        for (String body :
+                List.of(
+                        """
+                        {"input": "hello"}
+                        """,
+                        """
+                        {"instructions": null, "input": "hello"}
+                        """,
+                        """
+                        {"input": [{"role": "user", "content": "hello"}]}
+                        """,
+                        """
+                        {"instructions": null, "input": [{"role": "user", "content": "hello"}]}
+                        """)) {
+            assertEquals(expected, json(responsesInput(body)));
+        }
+        for (String body :
+                List.of(
+                        "{}",
+                        "{\"instructions\": null}",
+                        "{\"input\": null}",
+                        "{\"instructions\": null, \"input\": null}")) {
+            assertNull(responsesInput(body), body);
+        }
+    }
+
+    @Test
+    void openAiResponsesEmptyContentIsPreserved() {
+        assertEquals(json("[]"), json(responsesInput("{\"input\": []}")));
+        assertEquals(
+                json(
+                        """
+                        [{"role": "system", "content": ""}, {"role": "user", "content": ""}]
+                        """),
+                json(responsesInput("{\"instructions\": \"\", \"input\": \"\"}")));
+    }
+
+    @Test
+    void openAiNonResponsesEndpointsDoNotNormalizeInputOrInstructions() {
+        for (List<String> path :
+                List.of(
+                        List.<String>of(),
+                        List.of("v1", "embeddings"),
+                        List.of("v1", "moderations"),
+                        List.of("v1", "audio", "speech"),
+                        List.of("v1", "chat", "completions"),
+                        List.of("v1", "responses", "resp_1"),
+                        List.of("responses", "batches"),
+                        List.of("v1", "not-responses"))) {
+            assertEquals(
+                    json("[\"hello\", \"world\"]"),
+                    json(
+                            inputJsonForRequest(
+                                    InstrumentationSemConv.PROVIDER_NAME_OPENAI,
+                                    path,
+                                    "POST",
+                                    """
+                                    {"instructions": "be terse", "input": ["hello", "world"]}
+                                    """)),
+                    path.toString());
+        }
+        assertNull(
+                inputJsonForRequest(
+                        InstrumentationSemConv.PROVIDER_NAME_OPENAI,
+                        List.of("v1", "responses"),
+                        "GET",
+                        "{\"instructions\": \"be terse\", \"input\": \"hello\"}"));
+        assertEquals(
+                json(
+                        """
+                        [{"role": "user", "content": "chat message"}]
+                        """),
+                json(
+                        inputJsonForRequest(
+                                InstrumentationSemConv.PROVIDER_NAME_OPENAI,
+                                List.of("v1", "chat", "completions"),
+                                "POST",
+                                """
+                                {"messages": [{"role": "user", "content": "chat message"}],
+                                 "instructions": "be terse", "input": "hello"}
+                                """)));
     }
 
     @Test

btx/build.gradle

@@ -116,6 +116,7 @@ dependencies {
 test {
     useJUnitPlatform()
     workingDir = rootProject.projectDir
+    inputs.file(rootProject.file('scripts/spec-fetch.sh'))
 
     // Hand the isolated client classpaths to the test JVM (resolved lazily so configuration
     // time doesn't force dependency resolution).
@@ -146,6 +147,7 @@ tasks.register('fetchSpec', Exec) {
     description = 'Fetches braintrust-spec YAML files for the pinned ref.'
 
     inputs.property('specRef', specRef)
+    inputs.file(rootProject.file('scripts/spec-fetch.sh'))
     outputs.dir(specOutputDir)
 
     def outDir = specOutputDir.get().asFile