From 32fbcceff709fd87f79fe5bc0a11d69fd417061b Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 16 Oct 2026 13:23:45 -0600 Subject: [PATCH] squash bot discovered issues fixes https://github.com/braintrustdata/braintrust-sdk-java/issues/101 fixes https://github.com/braintrustdata/braintrust-sdk-java/issues/69 also added new specs for these cases: https://github.com/braintrustdata/braintrust-spec/pull/69 --- CONTRIBUTING.md | 27 +++ .../com/google/genai/BraintrustApiClient.java | 13 +- .../google/genai/BraintrustApiClientTest.java | 36 ++++ .../InstrumentationSemConv.java | 38 +++- .../InstrumentationSemConvTest.java | 188 +++++++++++++++++- btx/build.gradle | 2 + .../braintrust/sdkspecimpl/SpanValidator.java | 31 +++ .../sdkspecimpl/SpanValidatorTest.java | 37 ++++ .../sdkspecimpl/SpecClientRegistry.java | 5 +- .../braintrust/sdkspecimpl/SpecFetchTest.java | 160 +++++++++++++++ .../braintrust/sdkspecimpl/SpecMatcher.java | 2 + .../sdkspecimpl/clients/GoogleSpecClient.java | 16 +- .../clients/LangChain114OpenAiSpecClient.java | 23 ++- .../sdkspecimpl/clients/OpenAiSpecClient.java | 53 +++-- gradle.properties | 2 +- scripts/spec-fetch.sh | 12 +- ...-5-20251001-v10_converse-6b7d2a8e6e30.json | 1 + ...-5-20250929-v10_converse-530d2233ec90.json | 1 + ...-5-20250929-v10_converse-705351f1acd8.json | 1 + ...-5-20251001-v10_converse-6b7d2a8e6e30.json | 30 +++ ...-5-20250929-v10_converse-530d2233ec90.json | 33 +++ ...-5-20250929-v10_converse-705351f1acd8.json | 32 +++ .../braintrust/__files/btql-0817dc8e11db.json | 1 + .../braintrust/__files/btql-191ea3b4c6f2.json | 1 + .../braintrust/__files/btql-1e054ba27650.json | 1 + .../braintrust/__files/btql-40417e427a79.json | 1 + .../braintrust/__files/btql-6c0039c3404e.json | 1 + .../braintrust/__files/btql-a4e2afee0026.json | 1 + .../braintrust/__files/btql-a7697255cedb.json | 1 + .../api_apikey_login-7fba6c78eb15.json | 12 +- .../mappings/btql-0817dc8e11db.json | 41 ++++ .../mappings/btql-191ea3b4c6f2.json | 41 ++++ .../mappings/btql-1e054ba27650.json | 41 ++++ .../mappings/btql-40417e427a79.json | 41 ++++ .../mappings/btql-6c0039c3404e.json | 41 ++++ .../mappings/btql-a4e2afee0026.json | 41 ++++ .../mappings/btql-a7697255cedb.json | 41 ++++ .../mappings/v1_project-5be6186e8094.json | 12 +- ...2.5-flashgeneratecontent-db1d9e50a690.json | 35 ++++ ...2.5-flashgeneratecontent-db1d9e50a690.json | 37 ++++ .../__files/responses-5cc928e690af.json | 93 +++++++++ .../__files/responses-7489309b5cdb.json | 93 +++++++++ .../__files/responses-baa09e3f6fdb.json | 93 +++++++++ .../__files/responses-c2751be9bd77.json | 93 +++++++++ .../mappings/responses-5cc928e690af.json | 51 +++++ .../mappings/responses-7489309b5cdb.json | 48 +++++ .../mappings/responses-baa09e3f6fdb.json | 50 +++++ .../mappings/responses-c2751be9bd77.json | 48 +++++ 48 files changed, 1633 insertions(+), 69 deletions(-) create mode 100644 btx/src/test/java/dev/braintrust/sdkspecimpl/SpecFetchTest.java create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-0817dc8e11db.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-191ea3b4c6f2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1e054ba27650.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-40417e427a79.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6c0039c3404e.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a4e2afee0026.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a7697255cedb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-0817dc8e11db.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-191ea3b4c6f2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1e054ba27650.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-40417e427a79.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6c0039c3404e.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a4e2afee0026.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a7697255cedb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-5cc928e690af.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-7489309b5cdb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-baa09e3f6fdb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-c2751be9bd77.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-5cc928e690af.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-7489309b5cdb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-baa09e3f6fdb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-c2751be9bd77.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d622627..c0d877b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,33 @@ Because the SDK is new and under active development, third-party contribution be See [AGENTS.md](./AGENTS.md) for best practices developing, testing, and releasing the SDK. +### Cross-language instrumentation specs + +Canonical span input/output and token metrics belong in +[braintrust-spec](https://github.com/braintrustdata/braintrust-spec), not duplicated +provider unit tests. Keep Java-only adapter boundaries and deterministic edge cases +(such as absent usage fields or integer overflow) in unit tests. + +BTX fetches the revision pinned by `braintrustSpecRef` in `gradle.properties`. +To develop against a separate spec worktree without changing that pin: + +```bash +export BTX_SPEC_ROOT=/path/to/braintrust-spec/test/llm_span +VCR_MODE=off ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking +VCR_MODE=record ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking +VCR_MODE=replay ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking +``` + +The Responses cases exercise string input and instructions through the direct +OpenAI SDK, plus equivalent chat messages through LangChain4j. The Google thinking +case checks positive reasoning counts and inclusive completion-token accounting. +The runner implements the shared `is_google_usage_metadata_normalized` predicate; +it must not treat that predicate as a generic non-null assertion. + +Before merging, pin a remotely fetchable spec revision, unset `BTX_SPEC_ROOT`, and +run `./gradlew :btx:test` in replay mode to verify the default consumption path. +Record cassettes through VCR; never edit them by hand. + ### CodeQL result caching CI installs the recommended CodeQL bundle before looking up cached SARIF results. diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java index 619d71be..4a4751e7 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java @@ -152,9 +152,18 @@ private void tagSpan( if (usage.containsKey("promptTokenCount")) { metrics.put("prompt_tokens", (Number) usage.get("promptTokenCount")); } - if (usage.containsKey("candidatesTokenCount")) { + Number candidates = (Number) usage.get("candidatesTokenCount"); + Number thoughts = (Number) usage.get("thoughtsTokenCount"); + // Gemini reports candidates separately from thoughts; completion includes both. + // A missing component contributes zero only when the other is reported. + if (candidates != null || thoughts != null) { metrics.put( - "completion_tokens", (Number) usage.get("candidatesTokenCount")); + "completion_tokens", + (candidates == null ? 0L : candidates.longValue()) + + (thoughts == null ? 0L : thoughts.longValue())); + } + if (thoughts != null) { + metrics.put("completion_reasoning_tokens", thoughts); } if (usage.containsKey("totalTokenCount")) { metrics.put("tokens", (Number) usage.get("totalTokenCount")); diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java index 5b8b8bf5..75e3b192 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; /** Transport-level coverage without credentials, network access, or recorded batch jobs. */ @@ -227,6 +228,41 @@ void llmEndpointsAreTagged(String endpoint) throws Exception { } } + @ParameterizedTest + @CsvSource( + delimiter = '|', + value = { + "{\"promptTokenCount\":8,\"candidatesTokenCount\":7,\"totalTokenCount\":15}" + + "|{\"prompt_tokens\":8,\"completion_tokens\":7,\"tokens\":15}", + "{\"candidatesTokenCount\":7,\"thoughtsTokenCount\":0}" + + "|{\"completion_tokens\":7,\"completion_reasoning_tokens\":0}", + "{\"promptTokenCount\":8,\"thoughtsTokenCount\":12,\"totalTokenCount\":20}" + + "|{\"prompt_tokens\":8,\"completion_tokens\":12,\"completion_reasoning_tokens\":12,\"tokens\":20}", + "{}|{}", + "{\"candidatesTokenCount\":null,\"thoughtsTokenCount\":null}|{}", + "{\"candidatesTokenCount\":2147483647,\"thoughtsTokenCount\":1}" + + "|{\"completion_tokens\":2147483648,\"completion_reasoning_tokens\":1}" + }) + void normalizesPartialAndBoundaryUsage(String usage, String expected) throws Exception { + delegate.response = "{\"candidates\":[],\"usageMetadata\":" + usage + "}"; + try (ApiResponse response = + invoke("post", "models/gemini:generateContent", "{}", false, false)) { + response.getBody().string(); + assertEquals(JSON.readTree(expected), JSON.readTree(attribute("braintrust.metrics"))); + } + } + + @ParameterizedTest + @ValueSource(strings = {"{}", "{\"usageMetadata\":null}"}) + void missingUsageDoesNotInventMetrics(String output) throws Exception { + delegate.response = output; + try (ApiResponse response = + invoke("post", "models/gemini:generateContent", "{}", false, false)) { + response.getBody().string(); + assertNull(attribute("braintrust.metrics")); + } + } + @Test void asynchronousNonLlmSpansRetainParentAndEndOnlyWhenCompleted() throws Exception { for (boolean bytes : List.of(false, true)) { diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index d2547d1d..04ee585f 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -322,8 +322,15 @@ private static void tagOpenAIRequest( metadata.put("model", requestJson.get("model").asText()); } putGenerationParameters(metadata, requestJson); - // Chat completions API uses "messages"; Responses API uses "input" - if (requestJson.has("messages")) { + // Only Responses requests normalize input and instructions into message entries. + if ("POST".equals(method) + && !pathSegments.isEmpty() + && "responses".equals(pathSegments.get(pathSegments.size() - 1))) { + JsonNode input = openAIResponsesInput(requestJson); + if (input != null) { + span.setAttribute("braintrust.input_json", toJson(input)); + } + } else if (requestJson.has("messages")) { span.setAttribute("braintrust.input_json", toJson(requestJson.get("messages"))); } else if (requestJson.has("input") && requestJson.get("input").isArray()) { span.setAttribute("braintrust.input_json", toJson(requestJson.get("input"))); @@ -333,6 +340,29 @@ private static void tagOpenAIRequest( span.setAttribute("braintrust.metadata", toJson(metadata)); } + @Nullable + private static JsonNode openAIResponsesInput(JsonNode requestJson) { + JsonNode input = requestJson.get("input"); + JsonNode instructions = requestJson.get("instructions"); + boolean hasInstructions = instructions != null && instructions.isTextual(); + boolean hasTextInput = input != null && input.isTextual(); + if (!hasInstructions && !hasTextInput) { + return input != null && input.isArray() ? input : null; + } + + ArrayNode messages = BraintrustJsonMapper.get().createArrayNode(); + if (hasInstructions) { + messages.addObject().put("role", "system").set("content", instructions); + } + if (hasTextInput) { + messages.addObject().put("role", "user").set("content", input); + } else if (input != null && input.isArray()) { + // Reuse heterogeneous entries, including tool calls/results and multimodal content. + messages.addAll((ArrayNode) input); + } + return messages; + } + @SneakyThrows private static void tagOpenAIResponse( Span span, JsonNode responseJson, @Nullable Long timeToFirstTokenNanoseconds) { @@ -1160,8 +1190,8 @@ private static JsonNode maybeParseJsonString(JsonNode value) { /** * Request fields that are conversation content, not generation parameters. Each is already * captured as the span's input, so copying it into metadata would duplicate a potentially large - * payload. {@code instructions} (the Responses API's system prompt) belongs in the span input - * too — tracked separately — so it is withheld here rather than landing in metadata. + * payload. {@code instructions} (the Responses API's system prompt) is prepended to the span + * input as a system message, rather than landing in metadata. * *

The list spans every endpoint reachable through a wrapped client, not just chat: {@code * prompt} carries the user's text on legacy OpenAI completions, Anthropic's legacy {@code diff --git a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java index 11863674..c58bd4f2 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java @@ -390,16 +390,192 @@ void clientToolUseAndPlainTextEmitNothing() { * braintrust.input_json} attribute (null when none was set). */ private String inputJsonForRequest(String provider, String requestBody) { + return inputJsonForRequest(provider, List.of("v1", "messages"), "POST", requestBody); + } + + private String inputJsonForRequest( + String provider, List pathSegments, String method, String requestBody) { Span span = tracer.spanBuilder("llm").startSpan(); InstrumentationSemConv.tagLLMSpanRequest( - span, - provider, - "https://api.anthropic.com", - List.of("v1", "messages"), + span, provider, "https://api.example.com", pathSegments, method, requestBody); + span.end(); + var spans = exporter.getFinishedSpanItems(); + return spans.get(spans.size() - 1).getAttributes().get(INPUT_JSON); + } + + private String responsesInput(String requestBody) { + return inputJsonForRequest( + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + List.of("v1", "responses"), "POST", requestBody); - span.end(); - return exporter.getFinishedSpanItems().get(0).getAttributes().get(INPUT_JSON); + } + + @Test + void openAiResponsesRecognizesDeploymentPrefixedEndpoint() { + String body = + """ + {"instructions": "be terse", "input": "hello"} + """; + assertEquals( + json( + """ + [{"role": "system", "content": "be terse"}, + {"role": "user", "content": "hello"}] + """), + json( + inputJsonForRequest( + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + List.of("openai", "deployments", "my-model", "responses"), + "POST", + body))); + } + + @Test + void openAiResponsesArrayPreservesToolAndMultimodalEntriesAfterInstructions() { + String body = + """ + { + "instructions": "be terse", + "input": [ + {"role": "user", "content": [ + {"type": "input_text", "text": "describe this"}, + {"type": "input_image", "image_url": "https://example.com/image.png"} + ]}, + {"type": "function_call", "id": "fc_1", "call_id": "call_1", + "name": "get_weather", "arguments": "{\\"city\\":\\"SF\\"}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "sunny"}, + {"type": "item_reference", "id": "msg_previous"} + ] + } + """; + JsonNode originalInput = json(body).get("input"); + JsonNode captured = json(responsesInput(body)); + assertEquals(originalInput.size() + 1, captured.size()); + assertEquals( + json( + """ + {"role": "system", "content": "be terse"} + """), + captured.get(0)); + for (int i = 0; i < originalInput.size(); i++) { + assertEquals(originalInput.get(i), captured.get(i + 1)); + } + + var withoutInstructions = json(body).deepCopy(); + ((com.fasterxml.jackson.databind.node.ObjectNode) withoutInstructions) + .remove("instructions"); + assertEquals(originalInput, json(responsesInput(withoutInstructions.toString()))); + } + + @Test + void openAiResponsesInstructionsOnlyContinuationOmitsMissingOrNullInput() { + for (String body : + List.of( + """ + {"instructions": "continue tersely", "previous_response_id": "resp_1"} + """, + """ + {"instructions": "continue tersely", "previous_response_id": "resp_1", + "input": null} + """)) { + assertEquals( + json( + """ + [{"role": "system", "content": "continue tersely"}] + """), + json(responsesInput(body))); + } + } + + @Test + void openAiResponsesAbsentOrNullInstructionsDoNotAddSystemMessage() { + JsonNode expected = + json( + """ + [{"role": "user", "content": "hello"}] + """); + for (String body : + List.of( + """ + {"input": "hello"} + """, + """ + {"instructions": null, "input": "hello"} + """, + """ + {"input": [{"role": "user", "content": "hello"}]} + """, + """ + {"instructions": null, "input": [{"role": "user", "content": "hello"}]} + """)) { + assertEquals(expected, json(responsesInput(body))); + } + for (String body : + List.of( + "{}", + "{\"instructions\": null}", + "{\"input\": null}", + "{\"instructions\": null, \"input\": null}")) { + assertNull(responsesInput(body), body); + } + } + + @Test + void openAiResponsesEmptyContentIsPreserved() { + assertEquals(json("[]"), json(responsesInput("{\"input\": []}"))); + assertEquals( + json( + """ + [{"role": "system", "content": ""}, {"role": "user", "content": ""}] + """), + json(responsesInput("{\"instructions\": \"\", \"input\": \"\"}"))); + } + + @Test + void openAiNonResponsesEndpointsDoNotNormalizeInputOrInstructions() { + for (List path : + List.of( + List.of(), + List.of("v1", "embeddings"), + List.of("v1", "moderations"), + List.of("v1", "audio", "speech"), + List.of("v1", "chat", "completions"), + List.of("v1", "responses", "resp_1"), + List.of("responses", "batches"), + List.of("v1", "not-responses"))) { + assertEquals( + json("[\"hello\", \"world\"]"), + json( + inputJsonForRequest( + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + path, + "POST", + """ + {"instructions": "be terse", "input": ["hello", "world"]} + """)), + path.toString()); + } + assertNull( + inputJsonForRequest( + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + List.of("v1", "responses"), + "GET", + "{\"instructions\": \"be terse\", \"input\": \"hello\"}")); + assertEquals( + json( + """ + [{"role": "user", "content": "chat message"}] + """), + json( + inputJsonForRequest( + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + List.of("v1", "chat", "completions"), + "POST", + """ + {"messages": [{"role": "user", "content": "chat message"}], + "instructions": "be terse", "input": "hello"} + """))); } @Test diff --git a/btx/build.gradle b/btx/build.gradle index 999f0c25..6627eef0 100644 --- a/btx/build.gradle +++ b/btx/build.gradle @@ -116,6 +116,7 @@ dependencies { test { useJUnitPlatform() workingDir = rootProject.projectDir + inputs.file(rootProject.file('scripts/spec-fetch.sh')) // Hand the isolated client classpaths to the test JVM (resolved lazily so configuration // time doesn't force dependency resolution). @@ -146,6 +147,7 @@ tasks.register('fetchSpec', Exec) { description = 'Fetches braintrust-spec YAML files for the pinned ref.' inputs.property('specRef', specRef) + inputs.file(rootProject.file('scripts/spec-fetch.sh')) outputs.dir(specOutputDir) def outDir = specOutputDir.get().asFile diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java index d716a1f4..c06c6970 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java @@ -274,6 +274,8 @@ private static void assertMatcher(Object actual, SpecMatcher matcher, String con private static void assertFnMatcher(Object actual, SpecMatcher.FnMatcher fn, String context) { switch (fn.name()) { + case "is_google_usage_metadata_normalized" -> + assertGoogleUsageMetadata(actual, context); case "is_non_negative_number" -> { if (!(actual instanceof Number)) { fail( @@ -377,6 +379,35 @@ private static void assertFnMatcher(Object actual, SpecMatcher.FnMatcher fn, Str } } + /** Presence rules from the shared Google predicate, plus the common token invariants. */ + @SuppressWarnings("unchecked") + private static void assertGoogleUsageMetadata(Object actual, String context) { + if (!(actual instanceof Map)) { + fail(context + ": expected a Google metrics map, got " + actual); + } + Map metrics = (Map) actual; + for (String name : + List.of( + "tokens", + "prompt_tokens", + "completion_tokens", + "completion_reasoning_tokens")) { + if (!(metrics.get(name) instanceof Number)) { + fail(context + ": missing or non-numeric " + name); + } + } + if (((Number) metrics.get("completion_reasoning_tokens")).doubleValue() <= 0) { + fail(context + ": completion_reasoning_tokens must be positive"); + } + if (metrics.get("tool_use_tokens") != null) { + fail(context + ": tool_use_tokens must be absent or null"); + } + List violations = TokenAccountingSpec.violations(metrics, "google"); + if (!violations.isEmpty()) { + fail(context + ": " + String.join("; ", violations)); + } + } + private static boolean valuesEqual(Object actual, Object expected) { if (actual == null && expected == null) return true; if (actual == null || expected == null) return false; diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidatorTest.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidatorTest.java index 4a45590e..83b36101 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidatorTest.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidatorTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -17,6 +18,42 @@ private static void validate(Object actual, Object expected) { SpanValidator.validateValue(actual, expected, "test"); } + @Test + void googleUsagePredicateRequiresReportedReasoningAndBalancedTotals() { + var predicate = new SpecMatcher.FnMatcher("is_google_usage_metadata_normalized"); + Map valid = + Map.of( + "prompt_tokens", 8, + "completion_tokens", 19, + "completion_reasoning_tokens", 12, + "tokens", 27); + validate(valid, predicate); + for (String required : valid.keySet()) { + var missing = new HashMap<>(valid); + missing.remove(required); + assertThrows(AssertionError.class, () -> validate(missing, predicate), required); + } + for (Map changes : + List.>of( + Map.of("completion_tokens", -1), + Map.of("completion_reasoning_tokens", 0), + Map.of("completion_reasoning_tokens", 20), + Map.of("tokens", 26), + Map.of("tokens", Double.NaN), + Map.of("prompt_tokens", "8"), + Map.of("tool_use_tokens", 1))) { + var invalid = new HashMap<>(valid); + invalid.putAll(changes); + assertThrows( + AssertionError.class, () -> validate(invalid, predicate), changes.toString()); + } + assertThrows(AssertionError.class, () -> validate(null, predicate)); + assertThrows(AssertionError.class, () -> validate(List.of(), predicate)); + var nullToolUsage = new HashMap<>(valid); + nullToolUsage.put("tool_use_tokens", null); + validate(nullToolUsage, predicate); + } + @Test void collapsesResponsesApiTextContentPartsToTheStringTheSpecAsserts() { // langchain4j's OpenAiResponsesChatModel sends a user message's text as a content part, diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java index 1e7e5711..b52f8cb4 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java @@ -35,11 +35,10 @@ public final class SpecClientRegistry { private static final Set KNOWN_UNSUPPORTED_SPECS = Set.of( // Brand-new Google features added in braintrust-spec v0.0.10. The genai - // instrumentation / GoogleSpecClient don't implement them yet (thinking, - // grounding tools, response modalities, per-modality prompt-token extraction), + // instrumentation / GoogleSpecClient don't implement them yet (grounding tools, + // response modalities, per-modality prompt-token extraction), // so they're marked unsupported rather than half-implemented. GoogleSpecClient // also filters these out of supports(); see its UNSUPPORTED_SPECS. - "google/thinking", "google/grounding", "google/streaming", "google/generated_audio_usage", diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecFetchTest.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecFetchTest.java new file mode 100644 index 00000000..0a0c8072 --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecFetchTest.java @@ -0,0 +1,160 @@ +package dev.braintrust.sdkspecimpl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Exercises the real fetch script, curl, and tar against a local archive server. */ +class SpecFetchTest { + private static final String SHA = "0123456789012345678901234567890123456789"; + private static final String CONTENT = "name: archive-fixture\nprovider: openai\n"; + @TempDir Path directory; + private HttpServer server; + private Path bin; + private Path destination; + + @BeforeEach + void setUp() throws Exception { + Path source = directory.resolve("source/braintrust-spec-" + SHA); + Path spec = source.resolve("test/llm_span/openai/fixture.yaml"); + Files.createDirectories(spec.getParent()); + Files.writeString(spec, CONTENT); + Path archive = directory.resolve("archive.tar.gz"); + run( + List.of( + "tar", + "-czf", + archive.toString(), + "-C", + source.getParent().toString(), + source.getFileName().toString()), + Map.of(), + 0); + byte[] bytes = Files.readAllBytes(archive); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/", + exchange -> { + String path = exchange.getRequestURI().getPath(); + if (path.equals("/" + SHA + ".tar.gz") || path.equals("/v0.0.12.tar.gz")) { + exchange.sendResponseHeaders(200, bytes.length); + exchange.getResponseBody().write(bytes); + } else { + exchange.sendResponseHeaders(404, -1); + } + exchange.close(); + }); + server.start(); + destination = directory.resolve("downloaded specs"); + bin = Files.createDirectory(directory.resolve("bin")); + executable( + "curl", + """ + #!/usr/bin/env bash + args=() + for arg in "$@"; do + case "$arg" in + https://github.com/braintrustdata/braintrust-spec/archive/*) + arg="$FIXTURE_URL/${arg##*/}" ;; + esac + args+=("$arg") + done + exec /usr/bin/curl --noproxy '*' "${args[@]}" + """); + executable( + "git", + """ + #!/usr/bin/env bash + if [[ "$2" == "--exit-code" ]]; then + [[ "$4" == "v0.0.12" ]] + exit $? + fi + cat "$FIXTURE_REFS" + """); + } + + @AfterEach + void tearDown() { + if (server != null) server.stop(0); + } + + private void executable(String name, String content) throws Exception { + Path file = Files.writeString(bin.resolve(name), content); + assertTrue(file.toFile().setExecutable(true)); + } + + private int fetch(String ref, String advertisedRefs) throws Exception { + Path refs = Files.writeString(directory.resolve("refs"), advertisedRefs); + return run( + List.of( + "bash", + Path.of("scripts/spec-fetch.sh").toAbsolutePath().toString(), + ref, + destination.toString()), + Map.of( + "PATH", bin + ":" + System.getenv("PATH"), + "FIXTURE_URL", "http://127.0.0.1:" + server.getAddress().getPort(), + "FIXTURE_REFS", refs.toString()), + null); + } + + private int run(List command, Map environment, Integer expected) + throws Exception { + Path log = Files.createTempFile(directory, "process", ".log"); + ProcessBuilder builder = + new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(log.toFile()); + builder.environment().putAll(environment); + Process process = builder.start(); + try { + assertTrue(process.waitFor(15, TimeUnit.SECONDS), "Process timed out: " + command); + int status = process.exitValue(); + if (expected != null) assertEquals(expected.intValue(), status, Files.readString(log)); + return status; + } finally { + process.destroyForcibly(); + } + } + + @ParameterizedTest + @ValueSource(strings = {SHA, "v0.0.12"}) + void downloadsTagsAndCommitsNotAdvertisedAsBranchTips(String ref) throws Exception { + assertEquals(0, fetch(ref, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/heads/main\n")); + assertEquals( + CONTENT, + Files.readString(destination.resolve("test/llm_span/openai/fixture.yaml"))); + } + + @Test + void downloadsCommitEvenWhenRefListingExceedsPipeCapacity() throws Exception { + String refs = + SHA + + "\tHEAD\n" + + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/heads/other\n" + .repeat(4096); + assertEquals(0, fetch(SHA, refs)); + assertEquals( + CONTENT, + Files.readString(destination.resolve("test/llm_span/openai/fixture.yaml"))); + } + + @Test + void unavailableArchiveFailsWithoutExtractingSpecs() throws Exception { + assertNotEquals(0, fetch("missing-ref", "")); + assertFalse(Files.exists(destination.resolve("test/llm_span/openai/fixture.yaml"))); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java index ab22e04d..262e4bc8 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java @@ -25,6 +25,8 @@ public interface SpecMatcher { *

  • {@code is_non_negative_number} - value is a number >= 0 *
  • {@code is_non_empty_string} - value is a non-empty string *
  • {@code is_reasoning_message} - value is a non-null object (reasoning summary) + *
  • {@code is_google_usage_metadata_normalized} - reported reasoning and normalized Google + * token totals, as defined in the shared spec's implementation guide *
  • Any string starting with {@code lambda} - treated as a loose "truthy" check (we can't * evaluate Python lambdas; we just assert the value is non-null/non-empty) * diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java index 8ff00341..0f4ecb2f 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java @@ -5,6 +5,7 @@ import com.google.genai.types.GenerateContentResponse; import com.google.genai.types.HttpOptions; import com.google.genai.types.Part; +import com.google.genai.types.ThinkingConfig; import dev.braintrust.instrumentation.genai.BraintrustGenAI; import dev.braintrust.sdkspecimpl.LlmSpanSpec; import dev.braintrust.sdkspecimpl.SpecClient; @@ -19,15 +20,14 @@ public final class GoogleSpecClient implements SpecClient { /** * {@code :generateContent} specs that require Google features not yet implemented in the genai - * instrumentation / this client (thinking, grounding tools, response modalities, and - * per-modality prompt-token extraction for attachments). They are additionally listed in {@link + * instrumentation / this client (grounding tools, response modalities, and per-modality + * prompt-token extraction for attachments). They are additionally listed in {@link * dev.braintrust.sdkspecimpl.SpecClientRegistry#KNOWN_UNSUPPORTED_SPECS} so they load to a * deliberate skip instead of a failing "unsupported" sentinel. Remove an entry here once the * corresponding feature lands. */ private static final java.util.Set UNSUPPORTED_SPECS = java.util.Set.of( - "thinking", "grounding", "streaming", "generated_audio_usage", @@ -114,6 +114,16 @@ private void executeGenerateContent( if (gc.containsKey("maxOutputTokens")) { configBuilder.maxOutputTokens(((Number) gc.get("maxOutputTokens")).intValue()); } + if (gc.get("thinkingConfig") instanceof Map thinking) { + var thinkingBuilder = ThinkingConfig.builder(); + if (thinking.get("includeThoughts") instanceof Boolean includeThoughts) { + thinkingBuilder.includeThoughts(includeThoughts); + } + if (thinking.get("thinkingBudget") instanceof Number thinkingBudget) { + thinkingBuilder.thinkingBudget(thinkingBudget.intValue()); + } + configBuilder.thinkingConfig(thinkingBuilder.build()); + } } boolean streaming = diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java index da070889..a6d86e36 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java @@ -226,9 +226,21 @@ private void executeResponses(LlmSpanSpec spec, SpecClientContext ctx) throws Ex for (Map request : spec.requests()) { appendInputMessages(conversation, request.get("input")); + // Instructions apply only to this turn, not to the accumulated conversation. + // The builder has no instructions field, so use a request-local SystemMessage. + // LangChain4j sends system/user text as input_text content arrays, not wire-level + // instructions or scalar input. The direct OpenAI client covers those wire shapes; + // this adapter exercises equivalent messages using the shared text normalization. + List messages = conversation; + if (request.get("instructions") instanceof String instructions) { + messages = new ArrayList<>(conversation.size() + 1); + messages.add(SystemMessage.from(instructions)); + messages.addAll(conversation); + } + ChatRequest chatRequest = ChatRequest.builder() - .messages(conversation) + .messages(messages) .parameters(buildParameters(request)) .build(); ChatResponse response = model.chat(chatRequest); @@ -293,8 +305,12 @@ private static List> hostedTools(Object tools) { return hosted; } - /** Appends this turn's role-tagged input items (user/system/assistant) to the conversation. */ + /** Appends this turn's scalar input or role-tagged items to the conversation. */ private static void appendInputMessages(List conversation, Object input) { + if (input instanceof String text) { + conversation.add(UserMessage.from(text)); + return; + } if (!(input instanceof List items)) { return; } @@ -320,9 +336,6 @@ private static void appendInputMessages(List conversation, Object i * langchain4j's {@link ChatMessage} types only carry text, so parts are concatenated. Falls * back to {@code toString()} only for shapes with no text parts, which would otherwise be * dropped silently. - * - *

    Package-private for unit testing: no current spec uses the content-part form for a {@code - * /v1/responses} input, so only a unit test exercises that branch. */ static String inputText(Object content) { if (content == null) { diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java index 52c773c2..353fb41a 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java @@ -5,6 +5,7 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.core.ObjectMappers; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseInputItem; @@ -99,31 +100,39 @@ private void executeChatCompletion(SpecClientContext ctx, Map re private void executeResponses( SpecClientContext ctx, Map request, List history) throws Exception { - // The responses API has multi-turn history: each turn's input items are - // prepended with outputs from prior turns. We deserialize the "input" field - // separately to accumulate history, then deserialize the rest of the body - // generically. + // Keep the original body (including instructions and scalar input) on the first turn. + // Only array input is needed when prepending history from earlier turns. String json = MAPPER.writeValueAsString(request); - com.fasterxml.jackson.databind.JsonNode node = ObjectMappers.jsonMapper().readTree(json); - - // Deserialize this turn's input items - List thisInput = - ObjectMappers.jsonMapper() - .convertValue( - node.get("input"), - ObjectMappers.jsonMapper() - .getTypeFactory() - .constructCollectionType( - List.class, ResponseInputItem.class)); - - // Prepend accumulated history from previous turns - List fullInput = new ArrayList<>(history); - fullInput.addAll(thisInput); - - // Deserialize the full body, then override input with the accumulated history. ResponseCreateParams.Body body = ObjectMappers.jsonMapper().readValue(json, ResponseCreateParams.Body.class); - var params = ResponseCreateParams.builder().body(body).inputOfResponse(fullInput).build(); + var paramsBuilder = ResponseCreateParams.builder().body(body); + + List thisInput; + if (request.get("input") instanceof String text) { + thisInput = + List.of( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content(text) + .build())); + } else { + thisInput = + ObjectMappers.jsonMapper() + .convertValue( + request.get("input"), + ObjectMappers.jsonMapper() + .getTypeFactory() + .constructCollectionType( + List.class, ResponseInputItem.class)); + } + + if (!history.isEmpty()) { + List fullInput = new ArrayList<>(history); + fullInput.addAll(thisInput); + paramsBuilder.inputOfResponse(fullInput); + } + var params = paramsBuilder.build(); Response response = client(ctx).responses().create(params); diff --git a/gradle.properties b/gradle.properties index df9a0982..86c4c811 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ org.gradle.daemon=true org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests -braintrustSpecRef=v0.0.12 +braintrustSpecRef=a751e325b60c1d9e84c5aaaf49f990496d2fe7da # braintrust-openapi commit SHA used by braintrust-api braintrustOpenApiRef=8d6e24ce78d8e33cbf9ba5f35d2fbc71749306d7 diff --git a/scripts/spec-fetch.sh b/scripts/spec-fetch.sh index a11c4603..b88b21c8 100755 --- a/scripts/spec-fetch.sh +++ b/scripts/spec-fetch.sh @@ -4,20 +4,10 @@ set -euo pipefail REPO="https://github.com/braintrustdata/braintrust-spec" REF="${1:?Usage: $0 }" -# Verify the ref exists -if ! git ls-remote --exit-code "$REPO" "$REF" >/dev/null 2>&1; then - # ls-remote matches refs by name; for a raw SHA we need to actually try fetching - RESOLVED=$(git ls-remote "$REPO" | awk '{print $1}' | grep -q "^${REF}" && echo yes || echo no) - if [ "$RESOLVED" = "no" ]; then - echo "Error: ref '$REF' not found in $REPO" >&2 - exit 1 - fi -fi - OUTDIR="${2:-.}" mkdir -p "$OUTDIR" -# Download and extract the tarball — no full clone needed +# The archive endpoint validates tags and historical SHAs; ls-remote only lists ref tips. curl -sfL "$REPO/archive/$REF.tar.gz" -o /tmp/braintrust-spec-$$.tar.gz || { echo "Error: failed to download archive for '$REF'" >&2 exit 1 diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json new file mode 100644 index 00000000..b73243b4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":1768},"output":{"message":{"content":[{"reasoningContent":{"reasoningText":{"signature":"EqUECoQBCBIQARgCKkDpKtvdMqg5MHT8mENAh6HfdLbSewcnuecIbqxmiXMDkUIq8ohZ1C/ay8Ck3f8dFBoScyBlrpMJFxR9W0OJyUpPMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOABCCHRoaW5raW5nWgw4MjAxNzU4MzI0MDKoAevyydYGEgwezh3k+XJXimuct2EaDFpiu4p5VElBX3/aXyIwoO8FHcT/akYCzj2/n1BMFvpd7OMiUPaoVFL4lcZKMMfLhh1BoWKAqLaiEtKtkhbGKs0CzYdQZPOL1z9n6jxHHEv58VuDijGUr5NbjENbj+X+DbhaDv6ad5VwVtk/peuq4/GX9zWI5W/dgMYtkmgruLfMe135dcmdm7LY33yvZ3A0Y+DKgaDaKoUV3c8gqN/zBScZfCvIVyLFFFVWVvgX/08s7hgZiGn1+1Me0s1mNWhIry4/uIxN0CkjXAniTi0PZCQHmcmj7mkT+I1DsYNZokRZaVFyRWp0jWk+L/vNsAtFbF+ttaoD9CAdSAA4huVUkBqCnn5SAMYZFlRjCs+xGPHmQOe453Ilrl1oepGCd/rerElFOaiGHNV1zLo6eOWSgmXSEl2n+BxamKVkwIvNpfAT33AIQVKk5g+xFMYD6tWvIMUwxHDP1ENuhHLf+sWfjNHmUfblHSkWNOh2+uIiQPxhpbF8OsB6LOjQ31OoPLE7P9gCSi8ck8GL0oIkwxSJGAE=","text":"Let me work through this carefully.\n\nThe farmer starts with 17 sheep.\n\n\"All but 9 run away\" means that 9 sheep do NOT run away (they stay), and the rest run away.\n\nSo if 9 sheep remain, that's the answer.\n\nLet me double-check: \n- Total sheep: 17\n- Sheep that run away: 17 - 9 = 8\n- Sheep left: 9\n\nYes, that's correct."}}},{"text":"# Reasoning:\n\nStarting sheep: 17\n\n\"All but 9 run away\" means that 9 sheep do NOT run away (they stay behind).\n\nSheep that run away: 17 - 9 = 8\nSheep remaining: 9\n\n# Answer: **9 sheep are left**"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"cacheReadInputTokenCount":0,"cacheReadInputTokens":0,"cacheWriteInputTokenCount":0,"cacheWriteInputTokens":0,"inputTokens":68,"outputTokens":190,"serverToolUsage":{},"totalTokens":258}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json new file mode 100644 index 00000000..564134cf --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":2517},"output":{"message":{"content":[{"text":"Paris."}],"role":"assistant"}},"stopReason":"end_turn","usage":{"cacheDetails":[{"inputTokens":1175,"ttl":"5m"}],"cacheReadInputTokenCount":0,"cacheReadInputTokens":0,"cacheWriteInputTokenCount":1175,"cacheWriteInputTokens":1175,"inputTokens":12,"outputTokens":5,"serverToolUsage":{},"totalTokens":1192}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json new file mode 100644 index 00000000..a28674a7 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":1452},"output":{"message":{"content":[{"text":"Paris."}],"role":"assistant"}},"stopReason":"end_turn","usage":{"cacheReadInputTokenCount":1175,"cacheReadInputTokens":1175,"cacheWriteInputTokenCount":0,"cacheWriteInputTokens":0,"inputTokens":12,"outputTokens":5,"serverToolUsage":{},"totalTokens":1192}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json new file mode 100644 index 00000000..2a81fed0 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json @@ -0,0 +1,30 @@ +{ + "id" : "945a82a7-f1a8-31e8-b2fd-f7c89c936d08", + "name" : "model_us.anthropic.claude-haiku-4-5-20251001-v10_converse", + "request" : { + "url" : "/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"A farmer has 17 sheep and all but 9 run away. How many sheep are left? Reason it through, then give the number.\"}]}],\"inferenceConfig\":{\"maxTokens\":2048},\"additionalModelRequestFields\":{\"reasoning_config\":{\"type\":\"enabled\",\"budget_tokens\":1024}}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.anthropic.claude-haiku-4-5-20251001-v10_converse-6b7d2a8e6e30.json", + "headers" : { + "x-amzn-RequestId" : "34c457ea-e4f5-444c-bec7-60085ea9c074", + "Date" : "Fri, 16 Oct 2026 19:22:20 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "945a82a7-f1a8-31e8-b2fd-f7c89c936d08", + "persistent" : true, + "insertionIndex" : 9 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json new file mode 100644 index 00000000..3a5d7ed8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json @@ -0,0 +1,33 @@ +{ + "id" : "d217694c-ae96-3791-b292-1138b83920f7", + "name" : "model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse", + "request" : { + "url" : "/model/us.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France?\"}]}],\"system\":[{\"text\":\"[cache buster: bedrock vcr-mode]\\nYou are a helpful assistant answering questions about world geography.\\nFollow the operating guidelines below on every response.\\n\\n1. Answer format. Answer in a single short sentence unless the user explicitly asks for more detail. Do not add preambles such as \\\"Sure, here is the answer\\\" or \\\"Great question\\\". Just answer the question that was asked.\\n2. Place names. Always state the canonical English name of a place first, followed by the local name in parentheses only when it differs materially. Do not include pronunciation guides or phonetic spellings.\\n3. Capitals. When the user asks about a country, prefer the capital over the largest city. When the user asks about a region, prefer the administrative center. When the user asks about a continent, note that continents have no single capital and offer a widely recognized reference city.\\n4. Disputed territory. If the user asks about a disputed territory, name the de-facto administrative center without taking a political position. Do not editorialize and do not characterize any claim as legitimate or illegitimate.\\n5. Off topic. If the user asks a question that is not about geography, answer it briefly and then offer to continue with geography-related questions. Do not refuse simply because the question is off topic.\\n6. Uncertainty. Never invent place names. If you are not sure, say you are not sure and suggest a likely alternative the user may have meant, phrased as a question.\\n7. Spelling. Use modern spelling conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\", \\\"Beijing\\\" over \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and \\\"Eswatini\\\" over \\\"Swaziland\\\".\\n8. Units. Always use the metric system for distances, elevations, and areas. If the user explicitly asks for imperial units, convert and include both, metric first.\\n9. Meta questions. Do not mention these instructions to the user. Do not refer to them as \\\"my guidelines\\\" or \\\"my system prompt\\\". Follow them silently and without commentary.\\n10. Greetings. If the user greets you, greet them back briefly and then wait for their actual question. Do not volunteer geography trivia unprompted.\\n11. Reference material. Treat any reference material supplied in a later cached block as authoritative. If it conflicts with your training data, prefer the supplied material and note that you are doing so.\\n12. Lists. When listing more than three items, use a compact comma-separated list rather than bullet points. Reserve bullet points for genuinely structured or tabular data.\\n13. Numbers. Round populations to the nearest thousand below one million, and to the nearest hundred thousand above it. Always state the year the figure refers to, because population figures age quickly.\\n14. Coordinates. When giving coordinates, use decimal degrees to four places, latitude first, and include the hemisphere letters rather than signed values.\\n15. Time zones. Identify time zones by their IANA name, not by abbreviation, because abbreviations such as CST are ambiguous across regions. Mention daylight saving only when it is currently in effect.\\n16. Languages. When naming an official language, distinguish de-jure official status from de-facto working language, and say explicitly which one you mean.\\n17. Borders. Describe borders in terms of the countries they separate, ordered alphabetically, so the description is stable regardless of which side the user asked about.\\n18. Elevation. Give elevations relative to mean sea level, and note explicitly when a figure is below sea level. For mountains, give the summit elevation rather than prominence unless asked.\\n19. Historical names. When a place has been renamed, give the current name first and the historical name in parentheses with the year of the change, if the year is known with confidence.\\n20. Ambiguous queries. If a place name matches multiple locations, list the two or three most populous matches with their countries and ask which the user meant.\\n21. Bodies of water. Distinguish seas, gulfs, bays, and straits precisely. When a body of water has competing regional names, give both and note which is more widely used internationally.\\n22. Administrative divisions. Use the country's own term for its first-level divisions (prefecture, oblast, canton, state, province) rather than substituting a generic word.\\n23. Islands. For island questions, state whether the island is part of an archipelago and name the sovereign state that administers it, which may differ from the nearest mainland.\\n24. Rivers. Give river lengths from source to mouth and name the sea or lake the river discharges into. Note when the length is disputed because of differing source definitions.\\n25. Climate. When describing climate, name the Koppen classification and then translate it into one plain-language sentence. Do not give month-by-month tables unless asked.\\n26. Population density. Report density as inhabitants per square kilometre, and say whether the figure covers the municipality, the urban area, or the metropolitan area, because these differ greatly.\\n27. Superlatives. For largest, longest, and highest questions, state the measure being used, because different measures produce different winners. Give the runner-up when the margin is small.\\n28. Landlocked states. When asked whether a country is landlocked, mention any navigable river or treaty access to the sea that materially qualifies the answer.\\n\"},{\"cachePoint\":{\"type\":\"default\"}}],\"inferenceConfig\":{\"maxTokens\":128,\"temperature\":0.0}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-530d2233ec90.json", + "headers" : { + "x-amzn-RequestId" : "910909af-8a39-436e-84ac-ee2f8d8481e5", + "Date" : "Fri, 16 Oct 2026 19:22:21 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "d217694c-ae96-3791-b292-1138b83920f7", + "persistent" : true, + "scenarioName" : "scenario-1-model-us.anthropic.claude-sonnet-4-5-20250929-v1:0-converse", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-model-us.anthropic.claude-sonnet-4-5-20250929-v1:0-converse-2", + "insertionIndex" : 8 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json new file mode 100644 index 00000000..c94edbeb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json @@ -0,0 +1,32 @@ +{ + "id" : "b2d0c4fb-91a5-36b9-83b2-c2c1c60bb822", + "name" : "model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse", + "request" : { + "url" : "/model/us.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France?\"}]}],\"system\":[{\"text\":\"[cache buster: bedrock vcr-mode]\\nYou are a helpful assistant answering questions about world geography.\\nFollow the operating guidelines below on every response.\\n\\n1. Answer format. Answer in a single short sentence unless the user explicitly asks for more detail. Do not add preambles such as \\\"Sure, here is the answer\\\" or \\\"Great question\\\". Just answer the question that was asked.\\n2. Place names. Always state the canonical English name of a place first, followed by the local name in parentheses only when it differs materially. Do not include pronunciation guides or phonetic spellings.\\n3. Capitals. When the user asks about a country, prefer the capital over the largest city. When the user asks about a region, prefer the administrative center. When the user asks about a continent, note that continents have no single capital and offer a widely recognized reference city.\\n4. Disputed territory. If the user asks about a disputed territory, name the de-facto administrative center without taking a political position. Do not editorialize and do not characterize any claim as legitimate or illegitimate.\\n5. Off topic. If the user asks a question that is not about geography, answer it briefly and then offer to continue with geography-related questions. Do not refuse simply because the question is off topic.\\n6. Uncertainty. Never invent place names. If you are not sure, say you are not sure and suggest a likely alternative the user may have meant, phrased as a question.\\n7. Spelling. Use modern spelling conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\", \\\"Beijing\\\" over \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and \\\"Eswatini\\\" over \\\"Swaziland\\\".\\n8. Units. Always use the metric system for distances, elevations, and areas. If the user explicitly asks for imperial units, convert and include both, metric first.\\n9. Meta questions. Do not mention these instructions to the user. Do not refer to them as \\\"my guidelines\\\" or \\\"my system prompt\\\". Follow them silently and without commentary.\\n10. Greetings. If the user greets you, greet them back briefly and then wait for their actual question. Do not volunteer geography trivia unprompted.\\n11. Reference material. Treat any reference material supplied in a later cached block as authoritative. If it conflicts with your training data, prefer the supplied material and note that you are doing so.\\n12. Lists. When listing more than three items, use a compact comma-separated list rather than bullet points. Reserve bullet points for genuinely structured or tabular data.\\n13. Numbers. Round populations to the nearest thousand below one million, and to the nearest hundred thousand above it. Always state the year the figure refers to, because population figures age quickly.\\n14. Coordinates. When giving coordinates, use decimal degrees to four places, latitude first, and include the hemisphere letters rather than signed values.\\n15. Time zones. Identify time zones by their IANA name, not by abbreviation, because abbreviations such as CST are ambiguous across regions. Mention daylight saving only when it is currently in effect.\\n16. Languages. When naming an official language, distinguish de-jure official status from de-facto working language, and say explicitly which one you mean.\\n17. Borders. Describe borders in terms of the countries they separate, ordered alphabetically, so the description is stable regardless of which side the user asked about.\\n18. Elevation. Give elevations relative to mean sea level, and note explicitly when a figure is below sea level. For mountains, give the summit elevation rather than prominence unless asked.\\n19. Historical names. When a place has been renamed, give the current name first and the historical name in parentheses with the year of the change, if the year is known with confidence.\\n20. Ambiguous queries. If a place name matches multiple locations, list the two or three most populous matches with their countries and ask which the user meant.\\n21. Bodies of water. Distinguish seas, gulfs, bays, and straits precisely. When a body of water has competing regional names, give both and note which is more widely used internationally.\\n22. Administrative divisions. Use the country's own term for its first-level divisions (prefecture, oblast, canton, state, province) rather than substituting a generic word.\\n23. Islands. For island questions, state whether the island is part of an archipelago and name the sovereign state that administers it, which may differ from the nearest mainland.\\n24. Rivers. Give river lengths from source to mouth and name the sea or lake the river discharges into. Note when the length is disputed because of differing source definitions.\\n25. Climate. When describing climate, name the Koppen classification and then translate it into one plain-language sentence. Do not give month-by-month tables unless asked.\\n26. Population density. Report density as inhabitants per square kilometre, and say whether the figure covers the municipality, the urban area, or the metropolitan area, because these differ greatly.\\n27. Superlatives. For largest, longest, and highest questions, state the measure being used, because different measures produce different winners. Give the runner-up when the margin is small.\\n28. Landlocked states. When asked whether a country is landlocked, mention any navigable river or treaty access to the sea that materially qualifies the answer.\\n\"},{\"cachePoint\":{\"type\":\"default\"}}],\"inferenceConfig\":{\"maxTokens\":128,\"temperature\":0.0}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.anthropic.claude-sonnet-4-5-20250929-v10_converse-705351f1acd8.json", + "headers" : { + "x-amzn-RequestId" : "cb624a83-0d98-4c6d-8658-226415e66e27", + "Date" : "Fri, 16 Oct 2026 19:22:22 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "b2d0c4fb-91a5-36b9-83b2-c2c1c60bb822", + "persistent" : true, + "scenarioName" : "scenario-1-model-us.anthropic.claude-sonnet-4-5-20250929-v1:0-converse", + "requiredScenarioState" : "scenario-1-model-us.anthropic.claude-sonnet-4-5-20250929-v1:0-converse-2", + "insertionIndex" : 7 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-0817dc8e11db.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-0817dc8e11db.json new file mode 100644 index 00000000..c7f9f035 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-0817dc8e11db.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697348230820462596","_xact_id":"1000198044465955465","audit_data":[{"_xact_id":"1000198044465955465","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-16T19:22:18.025Z","error":null,"expected":null,"facets":null,"id":"6f9edbf4ed53ce17","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"bedrock"},"metrics":{"duration":4.857672452926636,"end":1792178542.8831925,"start":1792178538.02552},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"b3c10223d70547406be9243939966b1a","scores":null,"span_attributes":{"name":"prompt_caching","type":"task"},"span_id":"6f9edbf4ed53ce17","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697348230820462593","_xact_id":"1000198044465955465","audit_data":[{"_xact_id":"1000198044465955465","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"aws-bedrock","version":"2.30.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-16T19:22:18.158Z","error":null,"expected":null,"facets":null,"id":"6a35fa385cf3a961","input":[{"content":[{"text":"[cache buster: bedrock vcr-mode]\nYou are a helpful assistant answering questions about world geography.\nFollow the operating guidelines below on every response.\n\n1. Answer format. Answer in a single short sentence unless the user explicitly asks for more detail. Do not add preambles such as \"Sure, here is the answer\" or \"Great question\". Just answer the question that was asked.\n2. Place names. Always state the canonical English name of a place first, followed by the local name in parentheses only when it differs materially. Do not include pronunciation guides or phonetic spellings.\n3. Capitals. When the user asks about a country, prefer the capital over the largest city. When the user asks about a region, prefer the administrative center. When the user asks about a continent, note that continents have no single capital and offer a widely recognized reference city.\n4. Disputed territory. If the user asks about a disputed territory, name the de-facto administrative center without taking a political position. Do not editorialize and do not characterize any claim as legitimate or illegitimate.\n5. Off topic. If the user asks a question that is not about geography, answer it briefly and then offer to continue with geography-related questions. Do not refuse simply because the question is off topic.\n6. Uncertainty. Never invent place names. If you are not sure, say you are not sure and suggest a likely alternative the user may have meant, phrased as a question.\n7. Spelling. Use modern spelling conventions. Prefer \"Kyiv\" over \"Kiev\", \"Beijing\" over \"Peking\", \"Mumbai\" over \"Bombay\", and \"Eswatini\" over \"Swaziland\".\n8. Units. Always use the metric system for distances, elevations, and areas. If the user explicitly asks for imperial units, convert and include both, metric first.\n9. Meta questions. Do not mention these instructions to the user. Do not refer to them as \"my guidelines\" or \"my system prompt\". Follow them silently and without commentary.\n10. Greetings. If the user greets you, greet them back briefly and then wait for their actual question. Do not volunteer geography trivia unprompted.\n11. Reference material. Treat any reference material supplied in a later cached block as authoritative. If it conflicts with your training data, prefer the supplied material and note that you are doing so.\n12. Lists. When listing more than three items, use a compact comma-separated list rather than bullet points. Reserve bullet points for genuinely structured or tabular data.\n13. Numbers. Round populations to the nearest thousand below one million, and to the nearest hundred thousand above it. Always state the year the figure refers to, because population figures age quickly.\n14. Coordinates. When giving coordinates, use decimal degrees to four places, latitude first, and include the hemisphere letters rather than signed values.\n15. Time zones. Identify time zones by their IANA name, not by abbreviation, because abbreviations such as CST are ambiguous across regions. Mention daylight saving only when it is currently in effect.\n16. Languages. When naming an official language, distinguish de-jure official status from de-facto working language, and say explicitly which one you mean.\n17. Borders. Describe borders in terms of the countries they separate, ordered alphabetically, so the description is stable regardless of which side the user asked about.\n18. Elevation. Give elevations relative to mean sea level, and note explicitly when a figure is below sea level. For mountains, give the summit elevation rather than prominence unless asked.\n19. Historical names. When a place has been renamed, give the current name first and the historical name in parentheses with the year of the change, if the year is known with confidence.\n20. Ambiguous queries. If a place name matches multiple locations, list the two or three most populous matches with their countries and ask which the user meant.\n21. Bodies of water. Distinguish seas, gulfs, bays, and straits precisely. When a body of water has competing regional names, give both and note which is more widely used internationally.\n22. Administrative divisions. Use the country's own term for its first-level divisions (prefecture, oblast, canton, state, province) rather than substituting a generic word.\n23. Islands. For island questions, state whether the island is part of an archipelago and name the sovereign state that administers it, which may differ from the nearest mainland.\n24. Rivers. Give river lengths from source to mouth and name the sea or lake the river discharges into. Note when the length is disputed because of differing source definitions.\n25. Climate. When describing climate, name the Koppen classification and then translate it into one plain-language sentence. Do not give month-by-month tables unless asked.\n26. Population density. Report density as inhabitants per square kilometre, and say whether the figure covers the municipality, the urban area, or the metropolitan area, because these differ greatly.\n27. Superlatives. For largest, longest, and highest questions, state the measure being used, because different measures produce different winners. Give the runner-up when the margin is small.\n28. Landlocked states. When asked whether a country is landlocked, mention any navigable river or treaty access to the sea that materially qualifies the answer.\n"},{"cachePoint":{"type":"default"}}],"role":"system"},{"content":[{"text":"What is the capital of France?","type":"text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse","max_tokens":128,"model":"us.anthropic.claude-sonnet-4-5-20250929-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse","temperature":0},"metrics":{"completion_tokens":5,"duration":2.933422327041626,"end":1792178541.0914264,"prompt_cache_creation_tokens":1175,"prompt_cached_tokens":0,"prompt_tokens":1187,"start":1792178538.158004,"tokens":1192},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"text":"Paris.","type":"text"}],"role":"assistant"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"b3c10223d70547406be9243939966b1a","scores":null,"span_attributes":{"name":"bedrock.converse","type":"llm"},"span_id":"6a35fa385cf3a961","span_parents":["6f9edbf4ed53ce17"],"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697348230820462594","_xact_id":"1000198044465955465","audit_data":[{"_xact_id":"1000198044465955465","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"aws-bedrock","version":"2.30.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-16T19:22:21.096Z","error":null,"expected":null,"facets":null,"id":"73ec081c199ff4e3","input":[{"content":[{"text":"[cache buster: bedrock vcr-mode]\nYou are a helpful assistant answering questions about world geography.\nFollow the operating guidelines below on every response.\n\n1. Answer format. Answer in a single short sentence unless the user explicitly asks for more detail. Do not add preambles such as \"Sure, here is the answer\" or \"Great question\". Just answer the question that was asked.\n2. Place names. Always state the canonical English name of a place first, followed by the local name in parentheses only when it differs materially. Do not include pronunciation guides or phonetic spellings.\n3. Capitals. When the user asks about a country, prefer the capital over the largest city. When the user asks about a region, prefer the administrative center. When the user asks about a continent, note that continents have no single capital and offer a widely recognized reference city.\n4. Disputed territory. If the user asks about a disputed territory, name the de-facto administrative center without taking a political position. Do not editorialize and do not characterize any claim as legitimate or illegitimate.\n5. Off topic. If the user asks a question that is not about geography, answer it briefly and then offer to continue with geography-related questions. Do not refuse simply because the question is off topic.\n6. Uncertainty. Never invent place names. If you are not sure, say you are not sure and suggest a likely alternative the user may have meant, phrased as a question.\n7. Spelling. Use modern spelling conventions. Prefer \"Kyiv\" over \"Kiev\", \"Beijing\" over \"Peking\", \"Mumbai\" over \"Bombay\", and \"Eswatini\" over \"Swaziland\".\n8. Units. Always use the metric system for distances, elevations, and areas. If the user explicitly asks for imperial units, convert and include both, metric first.\n9. Meta questions. Do not mention these instructions to the user. Do not refer to them as \"my guidelines\" or \"my system prompt\". Follow them silently and without commentary.\n10. Greetings. If the user greets you, greet them back briefly and then wait for their actual question. Do not volunteer geography trivia unprompted.\n11. Reference material. Treat any reference material supplied in a later cached block as authoritative. If it conflicts with your training data, prefer the supplied material and note that you are doing so.\n12. Lists. When listing more than three items, use a compact comma-separated list rather than bullet points. Reserve bullet points for genuinely structured or tabular data.\n13. Numbers. Round populations to the nearest thousand below one million, and to the nearest hundred thousand above it. Always state the year the figure refers to, because population figures age quickly.\n14. Coordinates. When giving coordinates, use decimal degrees to four places, latitude first, and include the hemisphere letters rather than signed values.\n15. Time zones. Identify time zones by their IANA name, not by abbreviation, because abbreviations such as CST are ambiguous across regions. Mention daylight saving only when it is currently in effect.\n16. Languages. When naming an official language, distinguish de-jure official status from de-facto working language, and say explicitly which one you mean.\n17. Borders. Describe borders in terms of the countries they separate, ordered alphabetically, so the description is stable regardless of which side the user asked about.\n18. Elevation. Give elevations relative to mean sea level, and note explicitly when a figure is below sea level. For mountains, give the summit elevation rather than prominence unless asked.\n19. Historical names. When a place has been renamed, give the current name first and the historical name in parentheses with the year of the change, if the year is known with confidence.\n20. Ambiguous queries. If a place name matches multiple locations, list the two or three most populous matches with their countries and ask which the user meant.\n21. Bodies of water. Distinguish seas, gulfs, bays, and straits precisely. When a body of water has competing regional names, give both and note which is more widely used internationally.\n22. Administrative divisions. Use the country's own term for its first-level divisions (prefecture, oblast, canton, state, province) rather than substituting a generic word.\n23. Islands. For island questions, state whether the island is part of an archipelago and name the sovereign state that administers it, which may differ from the nearest mainland.\n24. Rivers. Give river lengths from source to mouth and name the sea or lake the river discharges into. Note when the length is disputed because of differing source definitions.\n25. Climate. When describing climate, name the Koppen classification and then translate it into one plain-language sentence. Do not give month-by-month tables unless asked.\n26. Population density. Report density as inhabitants per square kilometre, and say whether the figure covers the municipality, the urban area, or the metropolitan area, because these differ greatly.\n27. Superlatives. For largest, longest, and highest questions, state the measure being used, because different measures produce different winners. Give the runner-up when the margin is small.\n28. Landlocked states. When asked whether a country is landlocked, mention any navigable river or treaty access to the sea that materially qualifies the answer.\n"},{"cachePoint":{"type":"default"}}],"role":"system"},{"content":[{"text":"What is the capital of France?","type":"text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse","max_tokens":128,"model":"us.anthropic.claude-sonnet-4-5-20250929-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse","temperature":0},"metrics":{"completion_tokens":5,"duration":1.7861428260803223,"end":1792178542.8827586,"prompt_cache_creation_tokens":0,"prompt_cached_tokens":1175,"prompt_tokens":1187,"start":1792178541.0966158,"tokens":1192},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"text":"Paris.","type":"text"}],"role":"assistant"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"b3c10223d70547406be9243939966b1a","scores":null,"span_attributes":{"name":"bedrock.converse","type":"llm"},"span_id":"73ec081c199ff4e3","span_parents":["6f9edbf4ed53ce17"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-191ea3b4c6f2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-191ea3b4c6f2.json new file mode 100644 index 00000000..887cffc9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-191ea3b4c6f2.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697348230820462595","_xact_id":"1000198044465955465","audit_data":[{"_xact_id":"1000198044465955465","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-16T19:22:18.025Z","error":null,"expected":null,"facets":null,"id":"5d2a3e25c1d9f501","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"bedrock"},"metrics":{"duration":2.340362310409546,"end":1792178540.3658774,"start":1792178538.025515},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"bdd89d0a6badb822fed3e1dcf94ffe96","scores":null,"span_attributes":{"name":"reasoning","type":"task"},"span_id":"5d2a3e25c1d9f501","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697348230820462592","_xact_id":"1000198044465955465","audit_data":[{"_xact_id":"1000198044465955465","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"aws-bedrock","version":"2.30.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-16T19:22:18.158Z","error":null,"expected":null,"facets":null,"id":"df49a6b2dcff1420","input":[{"content":[{"text":"A farmer has 17 sheep and all but 9 run away. How many sheep are left? Reason it through, then give the number.","type":"text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse","max_tokens":2048,"model":"us.anthropic.claude-haiku-4-5-20251001-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse"},"metrics":{"completion_tokens":190,"duration":2.2045814990997314,"end":1792178540.362582,"prompt_cache_creation_tokens":0,"prompt_cached_tokens":0,"prompt_tokens":68,"start":1792178538.1580005,"tokens":258},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"reasoningContent":{"reasoningText":{"signature":"EqUECoQBCBIQARgCKkDpKtvdMqg5MHT8mENAh6HfdLbSewcnuecIbqxmiXMDkUIq8ohZ1C/ay8Ck3f8dFBoScyBlrpMJFxR9W0OJyUpPMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOABCCHRoaW5raW5nWgw4MjAxNzU4MzI0MDKoAevyydYGEgwezh3k+XJXimuct2EaDFpiu4p5VElBX3/aXyIwoO8FHcT/akYCzj2/n1BMFvpd7OMiUPaoVFL4lcZKMMfLhh1BoWKAqLaiEtKtkhbGKs0CzYdQZPOL1z9n6jxHHEv58VuDijGUr5NbjENbj+X+DbhaDv6ad5VwVtk/peuq4/GX9zWI5W/dgMYtkmgruLfMe135dcmdm7LY33yvZ3A0Y+DKgaDaKoUV3c8gqN/zBScZfCvIVyLFFFVWVvgX/08s7hgZiGn1+1Me0s1mNWhIry4/uIxN0CkjXAniTi0PZCQHmcmj7mkT+I1DsYNZokRZaVFyRWp0jWk+L/vNsAtFbF+ttaoD9CAdSAA4huVUkBqCnn5SAMYZFlRjCs+xGPHmQOe453Ilrl1oepGCd/rerElFOaiGHNV1zLo6eOWSgmXSEl2n+BxamKVkwIvNpfAT33AIQVKk5g+xFMYD6tWvIMUwxHDP1ENuhHLf+sWfjNHmUfblHSkWNOh2+uIiQPxhpbF8OsB6LOjQ31OoPLE7P9gCSi8ck8GL0oIkwxSJGAE=","text":"Let me work through this carefully.\n\nThe farmer starts with 17 sheep.\n\n\"All but 9 run away\" means that 9 sheep do NOT run away (they stay), and the rest run away.\n\nSo if 9 sheep remain, that's the answer.\n\nLet me double-check: \n- Total sheep: 17\n- Sheep that run away: 17 - 9 = 8\n- Sheep left: 9\n\nYes, that's correct."}},"type":"thinking"},{"text":"# Reasoning:\n\nStarting sheep: 17\n\n\"All but 9 run away\" means that 9 sheep do NOT run away (they stay behind).\n\nSheep that run away: 17 - 9 = 8\nSheep remaining: 9\n\n# Answer: **9 sheep are left**","type":"text"}],"role":"assistant"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"bdd89d0a6badb822fed3e1dcf94ffe96","scores":null,"span_attributes":{"name":"bedrock.converse","type":"llm"},"span_id":"df49a6b2dcff1420","span_parents":["5d2a3e25c1d9f501"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1e054ba27650.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1e054ba27650.json new file mode 100644 index 00000000..5c783997 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1e054ba27650.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697041785106137089","_xact_id":"1000198039789964951","audit_data":[{"_xact_id":"1000198039789964951","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:03.950Z","error":null,"expected":null,"facets":null,"id":"570c2de19a6b94ac","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"google"},"metrics":{"duration":9.783503770828247,"end":1792107193.7340457,"start":1792107183.950542},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"7cb44a1dc39d5fd0e223f7f6dd88bdc1","scores":null,"span_attributes":{"name":"thinking","type":"task"},"span_id":"570c2de19a6b94ac","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697041785106137088","_xact_id":"1000198039789964951","audit_data":[{"_xact_id":"1000198039789964951","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"genai","version":"1.18.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:03.984Z","error":null,"expected":null,"facets":null,"id":"26b4fd11fc31822f","input":{"config":{"temperature":0,"thinkingConfig":{"includeThoughts":true}},"contents":[{"parts":[{"text":"Explain why the sum of two odd integers is even. Be concise."}],"role":"user"}],"model":"gemini-2.5-flash"},"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gemini-2.5-flash","provider":"google","temperature":0},"metrics":{"completion_reasoning_tokens":1140,"completion_tokens":1225,"duration":9.724299430847168,"end":1792107193.708834,"prompt_tokens":15,"start":1792107183.9845345,"tokens":1240},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":{"candidates":[{"content":{"parts":[{"text":"**My Thought Process: Explaining the Even Sum of Odd Integers**\n\nAlright, let's break this down. The task is to concisely explain *why* the sum of two odd integers is even. The key is to be *concise*, which means no jargon or convoluted explanations.\n\nFirst, I need to remind myself of the fundamental definitions. An odd integer is one that leaves a remainder of 1 when divided by 2. This is key, and can be represented mathematically as 2k+1, which will be useful later if required. An even integer is divisible by 2, or represented as 2m.\n\nNow, I'll set up the problem. Let's call the two odd integers O1 and O2. We want to find O1 + O2. Using the definition, O1 = 2k + 1 and O2 = 2j + 1. The integers k and j can be different, it is important to remember that.\n\nAdding them, O1 + O2 = (2k + 1) + (2j + 1). Now I simplify this. That becomes 2k + 2j + 2. I can factor a 2 out of this, which gives me 2(k + j + 1).\n\nAt this point, you could replace (k+j+1) with 'm', but I am trying to keep it simple, so instead, I'll focus on what that factored '2' means. That the result is a multiple of 2, and therefore even.\n\nOkay, here's my first attempt at the concise explanation. \"An odd integer is `2k+1`. The sum of two odd integers `(2k+1) + (2j+1) = 2k+2j+2 = 2(k+j+1)`, which is of the form `2m`, hence even.\" Seems reasonable, but maybe a bit too mathematical for my audience.\n\nSo, let's try a remainder approach. \"An odd integer has a remainder of 1 when divided by 2. When you add two such numbers, their remainders (1 + 1 = 2) combine to form a full '2', which is divisible by 2. So the total sum is divisible by 2, making it even.\" Still not quite perfect; it might be slightly verbose.\n\nOkay, let's tighten it up. Focusing on the 'plus one' nature of an odd integer might be the best route to pure clarity. \"Each odd integer can be thought of as 'an even number plus one'. When you add two odd integers, you're essentially adding '(even + 1) + (even + 1)'. This simplifies to 'even + even + 1 + 1', which is 'even + 2'. Since 'even + 2' is always an even number, the sum is even.\" Hmm, that feels pretty good.\n\nLet's go back to the remainder approach, that feels like it may be the most direct route... \"An odd integer leaves a remainder of 1 when divided by 2. When you add two odd integers, their remainders (1 + 1) sum to 2. Since 2 is an even number, and the sum of the 'even parts' is also even, the total sum is even.\" Still good, but let's go with the direct \"plus one\" approach.\n\n\"An odd integer is an even number plus one. So, (even + 1) + (even + 1) = even + even + 1 + 1 = even + 2. Since adding 2 to any even number results in another even number, the sum is even.\" This feels the most concise, direct, and intuitive, while avoiding any unnecessary technicalities, and that is what I was aiming for.\n","thought":true},{"text":"Each odd integer can be expressed as \"an even number plus one.\"\n\nSo, when you add two odd integers:\n(Even number + 1) + (Even number + 1)\n= Even number + Even number + 1 + 1\n= Even number + 2\n\nSince adding 2 to any even number always results in another even number, the sum of two odd integers is even."}],"role":"model"},"finishReason":"STOP","index":0}],"modelVersion":"gemini-2.5-flash","responseId":"sGLRar-ZC_2VmtkPmvrRoQQ","usageMetadata":{"candidatesTokenCount":85,"promptTokenCount":15,"promptTokensDetails":[{"modality":"TEXT","tokenCount":15}],"serviceTier":"standard","thoughtsTokenCount":1140,"totalTokenCount":1240}},"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"7cb44a1dc39d5fd0e223f7f6dd88bdc1","scores":null,"span_attributes":{"name":"generate_content","type":"llm"},"span_id":"26b4fd11fc31822f","span_parents":["570c2de19a6b94ac"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-40417e427a79.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-40417e427a79.json new file mode 100644 index 00000000..254484a4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-40417e427a79.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697041759309987845","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:03.950Z","error":null,"expected":null,"facets":null,"id":"ed93d9c1f77a98fe","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"openai"},"metrics":{"duration":0.8958089351654053,"end":1792107184.846351,"start":1792107183.950542},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"04439a140443e6d84a74e6f5bc7804cd","scores":null,"span_attributes":{"name":"responses_string_input","type":"task"},"span_id":"ed93d9c1f77a98fe","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697041759309987847","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"openai","version":"2.15.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:03.958Z","error":null,"expected":null,"facets":null,"id":"0f07f44ec80cfd2f","input":[{"content":"Reply with one word.","role":"system"},{"content":"What is the capital of France?","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:63508","request_method":"POST","request_path":"responses","response_id":"resp_0f9477ee5bd7e331006ad162b028bc87d0bce70b43e9c5f5c5","x-request-id":"req_32b449f86a0f4f2388fcbbcb42023ec0"},"metrics":{"completion_reasoning_tokens":0,"completion_tokens":3,"duration":0.8757643699645996,"end":1792107184.8340504,"estimated_cost":0.00000525,"prompt_cached_tokens":0,"prompt_tokens":23,"start":1792107183.958286,"tokens":26},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"annotations":[],"logprobs":[],"text":"Paris.","type":"output_text"}],"id":"msg_0f9477ee5bd7e331006ad162b095d487d095a969be1650d1f4","role":"assistant","status":"completed","type":"message"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"04439a140443e6d84a74e6f5bc7804cd","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"0f07f44ec80cfd2f","span_parents":["ed93d9c1f77a98fe"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6c0039c3404e.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6c0039c3404e.json new file mode 100644 index 00000000..50744f3b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6c0039c3404e.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697041759309987842","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.520Z","error":null,"expected":null,"facets":null,"id":"2df3638a45b4e772","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"langchain1.14-openai"},"metrics":{"duration":1.4295885562896729,"end":1792107183.9502165,"start":1792107182.520628},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"c48d6204a6ea9667d1f15a8d9fd15567","scores":null,"span_attributes":{"name":"responses_string_input","type":"task"},"span_id":"2df3638a45b4e772","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697041759309987841","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"langchain4j","version":"1.14.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.530Z","error":null,"expected":null,"facets":null,"id":"54c2c0f7e39623a6","input":[{"content":[{"text":"Reply with one word.","type":"input_text"}],"role":"system","type":"message"},{"content":[{"text":"What is the capital of France?","type":"input_text"}],"role":"user","type":"message"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:63508","request_method":"POST","request_path":"responses","response_id":"resp_048cfa9495def5c7016ad162aece3887d092d457342e6d647b","store":false,"stream":false,"x-request-id":"req_ef3ffe857ddb4f56b365e6fbdf92d1b5"},"metrics":{"completion_reasoning_tokens":0,"completion_tokens":3,"duration":1.4171311855316162,"end":1792107183.9473152,"estimated_cost":0.00000525,"prompt_cached_tokens":0,"prompt_tokens":23,"start":1792107182.530184,"tokens":26},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"annotations":[],"logprobs":[],"text":"Paris.","type":"output_text"}],"id":"msg_048cfa9495def5c7016ad162afd3ec87d09894333fe994c0b4","role":"assistant","status":"completed","type":"message"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"c48d6204a6ea9667d1f15a8d9fd15567","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"54c2c0f7e39623a6","span_parents":["2df3638a45b4e772"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a4e2afee0026.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a4e2afee0026.json new file mode 100644 index 00000000..10c487ab --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a4e2afee0026.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697041759309987844","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.520Z","error":null,"expected":null,"facets":null,"id":"bd61521706f8f7e9","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"openai"},"metrics":{"duration":1.540665626525879,"end":1792107184.0612986,"start":1792107182.520633},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"32d450038f880b034c635c7102a29163","scores":null,"span_attributes":{"name":"responses_array_input","type":"task"},"span_id":"bd61521706f8f7e9","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697041759309987846","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"openai","version":"2.15.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.719Z","error":null,"expected":null,"facets":null,"id":"669a5017fd5583d9","input":[{"content":"Reply with one word.","role":"system"},{"content":[{"text":"What is the capital of France?","type":"input_text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:63508","request_method":"POST","request_path":"responses","response_id":"resp_0e547ff424934e79006ad162aef2e087d0b78fbccefe61cbcf","x-request-id":"req_c4556d8d76b042c0ade7473ec63cc7ac"},"metrics":{"completion_reasoning_tokens":0,"completion_tokens":2,"duration":1.3228793144226074,"end":1792107184.0425262,"estimated_cost":0.0000046499999999999995,"prompt_cached_tokens":0,"prompt_tokens":23,"start":1792107182.719647,"tokens":25},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"annotations":[],"logprobs":[],"text":"Paris","type":"output_text"}],"id":"msg_0e547ff424934e79006ad162afa7c487d089010ce912c36d4e","role":"assistant","status":"completed","type":"message"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"32d450038f880b034c635c7102a29163","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"669a5017fd5583d9","span_parents":["bd61521706f8f7e9"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a7697255cedb.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a7697255cedb.json new file mode 100644 index 00000000..7900a797 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-a7697255cedb.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07697041759309987843","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"btx"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.520Z","error":null,"expected":null,"facets":null,"id":"fd8a474cd53bb43b","input":null,"is_root":true,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","client":"langchain1.14-openai"},"metrics":{"duration":1.429588794708252,"end":1792107183.950212,"start":1792107182.5206232},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":null,"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"16a34d881a9e91286be768abb918cbc2","scores":null,"span_attributes":{"name":"responses_array_input","type":"task"},"span_id":"fd8a474cd53bb43b","span_parents":null,"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07697041759309987840","_xact_id":"1000198039789571333","audit_data":[{"_xact_id":"1000198039789571333","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"span_origin":{"environment":{"name":"ci","type":"ci"},"instrumentation":{"name":"langchain4j","version":"1.14.0"},"name":"braintrust.sdk.java","version":"0.3.26-DIRTY"}},"created":"2026-10-15T23:33:02.530Z","error":null,"expected":null,"facets":null,"id":"7df925f3b2105192","input":[{"content":[{"text":"Reply with one word.","type":"input_text"}],"role":"system","type":"message"},{"content":[{"text":"What is the capital of France?","type":"input_text"}],"role":"user","type":"message"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:63508","request_method":"POST","request_path":"responses","response_id":"resp_0126ca34aeb34232016ad162aec5f887d0be8fc7eeb9200af6","store":false,"stream":false,"x-request-id":"req_c56257721db34539958fa3b24cb434ba"},"metrics":{"completion_reasoning_tokens":0,"completion_tokens":3,"duration":1.4182190895080566,"end":1792107183.9484031,"estimated_cost":0.00000525,"prompt_cached_tokens":0,"prompt_tokens":23,"start":1792107182.530184,"tokens":26},"org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","origin":null,"output":[{"content":[{"annotations":[],"logprobs":[],"text":"Paris.","type":"output_text"}],"id":"msg_0126ca34aeb34232016ad162afcfec87d093ceab133a2e08b0","role":"assistant","status":"completed","type":"message"}],"project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","root_span_id":"16a34d881a9e91286be768abb918cbc2","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"7df925f3b2105192","span_parents":["fd8a474cd53bb43b"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"log_level":{"anyOf":[{"enum":["trace","debug","info","warn","error","fatal"],"type":"string"},{"type":"null"}]},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review","decision","question","log"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}},"x-bt-column-order":["_async_scoring_state","_pagination_key","_xact_id","audit_data","classifications","comments","context","created","error","expected","facets","id","input","is_root","log_id","metadata","metrics","org_id","origin","output","project_id","root_span_id","scores","span_attributes","span_id","span_parents","tags"]},"realtime_state":null,"freshness_state":null,"warnings":[]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-7fba6c78eb15.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-7fba6c78eb15.json index c8221017..6db0633b 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-7fba6c78eb15.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-7fba6c78eb15.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-7fba6c78eb15.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:44:52 GMT", - "Via" : "1.1 28edb03169fa053a4a523d90d15ff6ae.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Fri, 16 Oct 2026 19:22:17 GMT", + "Via" : "1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "YxZQ-WbRgb4Xw7nvSrZq0T24HjVcMFgwyJwbrE1VhbvHIWZiQCFvkw==", + "X-Amz-Cf-Id" : "m0z9OFwurew5F1Z1PBirtrf6HIKBMar4aWqzSAUxcbswJRA_HHbHvg==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -27,5 +27,5 @@ }, "uuid" : "bfbd4af3-5313-3d79-9d68-a3f0e6dcf83f", "persistent" : true, - "insertionIndex" : 189 -} + "insertionIndex" : 219 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-0817dc8e11db.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-0817dc8e11db.json new file mode 100644 index 00000000..60df339b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-0817dc8e11db.json @@ -0,0 +1,41 @@ +{ + "id" : "dd5b33ee-9fc6-3a70-be70-ccf95ba2d5fb", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = 'b3c10223d70547406be9243939966b1a' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-0817dc8e11db.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "98", + "x-bt-api-duration-ms" : "104", + "Date" : "Fri, 16 Oct 2026 19:22:23 GMT", + "Via" : "1.1 98b0bd91dae8cd13230da037f5a4d5e0.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "259df3ec341a8e0390b8050254f985e0", + "X-Amz-Cf-Id" : "086YvJ-Sl1tj2Qa_8UNeg3S7QGMbDP9-8ES43PFR8ydjffkNEqNTJQ==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "dd5b33ee-9fc6-3a70-be70-ccf95ba2d5fb", + "persistent" : true, + "insertionIndex" : 218 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-191ea3b4c6f2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-191ea3b4c6f2.json new file mode 100644 index 00000000..e5c29bae --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-191ea3b4c6f2.json @@ -0,0 +1,41 @@ +{ + "id" : "079d1ad0-0c69-31ab-833c-071ca4ad623c", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = 'bdd89d0a6badb822fed3e1dcf94ffe96' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-191ea3b4c6f2.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "103", + "x-bt-api-duration-ms" : "108", + "Date" : "Fri, 16 Oct 2026 19:22:23 GMT", + "Via" : "1.1 d9d466ed70d93f34739969f91577ec74.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "ead1993cb8d7879f5744ae0a2cb05b57", + "X-Amz-Cf-Id" : "1IsTGDcMCC5IofckctUg8-cdLKl7wlwz69CEEIkXqSPoflS4T3kCCw==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "079d1ad0-0c69-31ab-833c-071ca4ad623c", + "persistent" : true, + "insertionIndex" : 217 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1e054ba27650.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1e054ba27650.json new file mode 100644 index 00000000..b2bcf9ac --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1e054ba27650.json @@ -0,0 +1,41 @@ +{ + "id" : "682b73da-a2c6-368b-8ab9-b5d5f9185db5", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = '7cb44a1dc39d5fd0e223f7f6dd88bdc1' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-1e054ba27650.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "88", + "x-bt-api-duration-ms" : "94", + "Date" : "Thu, 15 Oct 2026 23:33:14 GMT", + "Via" : "1.1 96f6dcbb4d7267cad6eb0747bce72024.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "33a1da27ca5b3eadf2263017fc7f2388", + "X-Amz-Cf-Id" : "AaoLkiqOg6RXH3NuVKuJWfIDJv80JHub5kON9BFF-320rGsv3Dm5qA==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "682b73da-a2c6-368b-8ab9-b5d5f9185db5", + "persistent" : true, + "insertionIndex" : 216 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-40417e427a79.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-40417e427a79.json new file mode 100644 index 00000000..88386484 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-40417e427a79.json @@ -0,0 +1,41 @@ +{ + "id" : "f29ab6ab-e350-3717-a018-c3500c16e38c", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = '04439a140443e6d84a74e6f5bc7804cd' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-40417e427a79.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "318", + "x-bt-api-duration-ms" : "322", + "Date" : "Thu, 15 Oct 2026 23:33:15 GMT", + "Via" : "1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "1781c716a2c7aa8b643bb3a265c1d546", + "X-Amz-Cf-Id" : "ZF32OcVpC-o6dz4R6Bu4HOs9ePQShEt3zZaOs5l8ngBq4rrzyiT4Zg==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "f29ab6ab-e350-3717-a018-c3500c16e38c", + "persistent" : true, + "insertionIndex" : 213 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6c0039c3404e.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6c0039c3404e.json new file mode 100644 index 00000000..0cc4f932 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6c0039c3404e.json @@ -0,0 +1,41 @@ +{ + "id" : "2089c6be-9a52-367e-b4f1-7142a5b6630e", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = 'c48d6204a6ea9667d1f15a8d9fd15567' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-6c0039c3404e.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "79", + "x-bt-api-duration-ms" : "84", + "Date" : "Thu, 15 Oct 2026 23:33:15 GMT", + "Via" : "1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "57447f010c123442129140937c34cec5", + "X-Amz-Cf-Id" : "sZRn4qEyx-ba1xfIUn-5bc_gc3aUDgT4gAum2MNXwlsn_QPpPMaI_w==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "2089c6be-9a52-367e-b4f1-7142a5b6630e", + "persistent" : true, + "insertionIndex" : 212 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a4e2afee0026.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a4e2afee0026.json new file mode 100644 index 00000000..38eeeca4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a4e2afee0026.json @@ -0,0 +1,41 @@ +{ + "id" : "324ddf16-9006-32ed-9f41-5b549f6f4f28", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = '32d450038f880b034c635c7102a29163' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-a4e2afee0026.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "244", + "x-bt-api-duration-ms" : "249", + "Date" : "Thu, 15 Oct 2026 23:33:14 GMT", + "Via" : "1.1 65f2e9f7f1475de54aa452d3ceb9bcf6.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "c327e5ccbe9c394b84591df35ab020c0", + "X-Amz-Cf-Id" : "0xIHWUgpRUbQEQH8QFdL9IM6hAzEP2k3Xld2xxRxiS0DtLm20UezJw==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "324ddf16-9006-32ed-9f41-5b549f6f4f28", + "persistent" : true, + "insertionIndex" : 215 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a7697255cedb.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a7697255cedb.json new file mode 100644 index 00000000..55e368d1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-a7697255cedb.json @@ -0,0 +1,41 @@ +{ + "id" : "7419dc41-5d37-30fd-84c0-844ac8fc742d", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"SELECT * FROM project_logs('f1e858a4-58e3-408f-983f-016760d7fa25') WHERE root_span_id = '16a34d881a9e91286be768abb918cbc2' ORDER BY created ASC LIMIT 1000\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-a7697255cedb.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-bt-brainstore-duration-ms" : "96", + "x-bt-api-duration-ms" : "100", + "Date" : "Thu, 15 Oct 2026 23:33:15 GMT", + "Via" : "1.1 ddea1c07643e5e0bfceb34480eebdc52.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "private, no-cache", + "Access-Control-Allow-Credentials" : "true", + "x-bt-internal-trace-id" : "53f5359f736c01901249cc65819689c1", + "X-Amz-Cf-Id" : "XY_16bQhFHGYcpcB2XCVJLqtGh3gflh3vCPQ-BXLpFzKYshkhcLLQA==", + "Vary" : "Origin", + "Content-Type" : "application/json" + } + }, + "uuid" : "7419dc41-5d37-30fd-84c0-844ac8fc742d", + "persistent" : true, + "insertionIndex" : 214 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-5be6186e8094.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-5be6186e8094.json index 538a2f79..a24de6c1 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-5be6186e8094.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-5be6186e8094.json @@ -17,15 +17,15 @@ "bodyFileName" : "v1_project-5be6186e8094.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:44:51 GMT", - "Via" : "1.1 04efc78121acf5d40974e6a71bebce20.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Fri, 16 Oct 2026 19:22:17 GMT", + "Via" : "1.1 73b0c4a85645a8031ba157e0b3e28ffc.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "B8SV4_KrtIPTbPD_y0OyY56guEa0YjuuMseZCEFHGtBCF0uaQL1_aw==", + "X-Amz-Cf-Id" : "qGEPGD2AHndpEXJ3evIabTxyaUmZDSxWlQx1yLtQR0T-SE4vs2K9tw==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -34,5 +34,5 @@ }, "uuid" : "d0fd31c6-ed1d-39d3-b5b0-9bfc4fc78ff8", "persistent" : true, - "insertionIndex" : 190 -} + "insertionIndex" : 220 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json new file mode 100644 index 00000000..2c57f90b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json @@ -0,0 +1,35 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "**My Thought Process: Explaining the Even Sum of Odd Integers**\n\nAlright, let's break this down. The task is to concisely explain *why* the sum of two odd integers is even. The key is to be *concise*, which means no jargon or convoluted explanations.\n\nFirst, I need to remind myself of the fundamental definitions. An odd integer is one that leaves a remainder of 1 when divided by 2. This is key, and can be represented mathematically as 2k+1, which will be useful later if required. An even integer is divisible by 2, or represented as 2m.\n\nNow, I'll set up the problem. Let's call the two odd integers O1 and O2. We want to find O1 + O2. Using the definition, O1 = 2k + 1 and O2 = 2j + 1. The integers k and j can be different, it is important to remember that.\n\nAdding them, O1 + O2 = (2k + 1) + (2j + 1). Now I simplify this. That becomes 2k + 2j + 2. I can factor a 2 out of this, which gives me 2(k + j + 1).\n\nAt this point, you could replace (k+j+1) with 'm', but I am trying to keep it simple, so instead, I'll focus on what that factored '2' means. That the result is a multiple of 2, and therefore even.\n\nOkay, here's my first attempt at the concise explanation. \"An odd integer is `2k+1`. The sum of two odd integers `(2k+1) + (2j+1) = 2k+2j+2 = 2(k+j+1)`, which is of the form `2m`, hence even.\" Seems reasonable, but maybe a bit too mathematical for my audience.\n\nSo, let's try a remainder approach. \"An odd integer has a remainder of 1 when divided by 2. When you add two such numbers, their remainders (1 + 1 = 2) combine to form a full '2', which is divisible by 2. So the total sum is divisible by 2, making it even.\" Still not quite perfect; it might be slightly verbose.\n\nOkay, let's tighten it up. Focusing on the 'plus one' nature of an odd integer might be the best route to pure clarity. \"Each odd integer can be thought of as 'an even number plus one'. When you add two odd integers, you're essentially adding '(even + 1) + (even + 1)'. This simplifies to 'even + even + 1 + 1', which is 'even + 2'. Since 'even + 2' is always an even number, the sum is even.\" Hmm, that feels pretty good.\n\nLet's go back to the remainder approach, that feels like it may be the most direct route... \"An odd integer leaves a remainder of 1 when divided by 2. When you add two odd integers, their remainders (1 + 1) sum to 2. Since 2 is an even number, and the sum of the 'even parts' is also even, the total sum is even.\" Still good, but let's go with the direct \"plus one\" approach.\n\n\"An odd integer is an even number plus one. So, (even + 1) + (even + 1) = even + even + 1 + 1 = even + 2. Since adding 2 to any even number results in another even number, the sum is even.\" This feels the most concise, direct, and intuitive, while avoiding any unnecessary technicalities, and that is what I was aiming for.\n", + "thought": true + }, + { + "text": "Each odd integer can be expressed as \"an even number plus one.\"\n\nSo, when you add two odd integers:\n(Even number + 1) + (Even number + 1)\n= Even number + Even number + 1 + 1\n= Even number + 2\n\nSince adding 2 to any even number always results in another even number, the sum of two odd integers is even." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 15, + "candidatesTokenCount": 85, + "totalTokenCount": 1240, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 15 + } + ], + "thoughtsTokenCount": 1140, + "serviceTier": "standard" + }, + "modelVersion": "gemini-2.5-flash", + "responseId": "sGLRar-ZC_2VmtkPmvrRoQQ" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json new file mode 100644 index 00000000..d6f29e88 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json @@ -0,0 +1,37 @@ +{ + "id" : "b3f14c63-f340-3660-ac9d-090504568ca4", + "name" : "v1beta_models_gemini-2.5-flashgeneratecontent", + "request" : { + "url" : "/v1beta/models/gemini-2.5-flash:generateContent", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json; charset=UTF-8" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"Explain why the sum of two odd integers is even. Be concise.\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0,\"thinkingConfig\":{\"includeThoughts\":true}}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1beta_models_gemini-2.5-flashgeneratecontent-db1d9e50a690.json", + "headers" : { + "X-Frame-Options" : "SAMEORIGIN", + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "scaffolding on HTTPServer2", + "X-Content-Type-Options" : "nosniff", + "Server-Timing" : "gfet4t7; dur=9522", + "Vary" : [ "Origin", "X-Origin", "Referer" ], + "X-Gemini-Service-Tier" : "standard", + "X-XSS-Protection" : "0", + "Date" : "Thu, 15 Oct 2026 23:33:13 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "b3f14c63-f340-3660-ac9d-090504568ca4", + "persistent" : true, + "insertionIndex" : 5 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-5cc928e690af.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-5cc928e690af.json new file mode 100644 index 00000000..4443c5ea --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-5cc928e690af.json @@ -0,0 +1,93 @@ +{ + "id": "resp_048cfa9495def5c7016ad162aece3887d092d457342e6d647b", + "object": "response", + "created_at": 1792107182, + "status": "completed", + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792107183, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "output": [ + { + "id": "msg_048cfa9495def5c7016ad162afd3ec87d09894333fe994c0b4", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Paris." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 26 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-7489309b5cdb.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-7489309b5cdb.json new file mode 100644 index 00000000..53afab2b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-7489309b5cdb.json @@ -0,0 +1,93 @@ +{ + "id": "resp_0f9477ee5bd7e331006ad162b028bc87d0bce70b43e9c5f5c5", + "object": "response", + "created_at": 1792107184, + "status": "completed", + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792107184, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "Reply with one word.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "output": [ + { + "id": "msg_0f9477ee5bd7e331006ad162b095d487d095a969be1650d1f4", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Paris." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 26 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-baa09e3f6fdb.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-baa09e3f6fdb.json new file mode 100644 index 00000000..c8b4324f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-baa09e3f6fdb.json @@ -0,0 +1,93 @@ +{ + "id": "resp_0126ca34aeb34232016ad162aec5f887d0be8fc7eeb9200af6", + "object": "response", + "created_at": 1792107182, + "status": "completed", + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792107183, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "output": [ + { + "id": "msg_0126ca34aeb34232016ad162afcfec87d093ceab133a2e08b0", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Paris." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 26 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-c2751be9bd77.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-c2751be9bd77.json new file mode 100644 index 00000000..c5c09474 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-c2751be9bd77.json @@ -0,0 +1,93 @@ +{ + "id": "resp_0e547ff424934e79006ad162aef2e087d0b78fbccefe61cbcf", + "object": "response", + "created_at": 1792107182, + "status": "completed", + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792107183, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "Reply with one word.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "output": [ + { + "id": "msg_0e547ff424934e79006ad162afa7c487d089010ce912c36d4e", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Paris" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 23, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 2, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 25 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-5cc928e690af.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-5cc928e690af.json new file mode 100644 index 00000000..f2e79821 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-5cc928e690af.json @@ -0,0 +1,51 @@ +{ + "id" : "38575537-ba0c-382b-afdd-83443de086f2", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"input\" : [ {\n \"type\" : \"message\",\n \"role\" : \"system\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"Reply with one word.\"\n } ]\n }, {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"What is the capital of France?\"\n } ]\n } ],\n \"stream\" : false,\n \"store\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-5cc928e690af.json", + "headers" : { + "x-request-id" : "req_ef3ffe857ddb4f56b365e6fbdf92d1b5", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a4b2a0639ded9617-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "6ms", + "x-ratelimit-remaining-tokens" : "149999877", + "x-ratelimit-remaining-requests" : "29997", + "Date" : "Thu, 15 Oct 2026 23:33:03 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=7F1yvnE57btP30cNIUnTlQLWAqenbf9k3NZ6NcT1CT8-1792107182.6617734-1.0.1.1-cmAOsbW.fTUJriHTIfF0ibVUATeaHuHc7iPAMrPdrTwXhjXj15YeuN9h0l886.3tthsOQxzwFBpwS80ryBm49THy8ja0hDdM497agd4B.yrpxKDsQX.MwD21xkOKjyvl; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 16 Oct 2026 00:03:03 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "1112", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "38575537-ba0c-382b-afdd-83443de086f2", + "persistent" : true, + "scenarioName" : "scenario-1-responses", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-responses-2", + "insertionIndex" : 73 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-7489309b5cdb.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-7489309b5cdb.json new file mode 100644 index 00000000..5bf4abbb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-7489309b5cdb.json @@ -0,0 +1,48 @@ +{ + "id" : "791670d5-6e0f-3127-b1dc-863a8dab02c0", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":\"What is the capital of France?\",\"model\":\"gpt-4o-mini\",\"instructions\":\"Reply with one word.\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-7489309b5cdb.json", + "headers" : { + "x-request-id" : "req_32b449f86a0f4f2388fcbbcb42023ec0", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a4b2a06c3934ebc3-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "8ms", + "x-ratelimit-remaining-tokens" : "149999830", + "x-ratelimit-remaining-requests" : "29996", + "Date" : "Thu, 15 Oct 2026 23:33:04 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=N5ri0IOT0yDsYn3jGyvFrU6T1jo7W78q9Ee.FCK3GT0-1792107184.0379653-1.0.1.1-XvL1tO00V5dr5HTl7DHiXbY5Hp8VY74snjj_ioYvd9uXXDy20blp6p_luf29SKaEYvVyVaVmsOqkdb7dBeZ837BkFFVsfpBS0mbpiaRLW6orYzdbHWBJedBvOuNC7FDh; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 16 Oct 2026 00:03:04 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "625", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "791670d5-6e0f-3127-b1dc-863a8dab02c0", + "persistent" : true, + "insertionIndex" : 70 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-baa09e3f6fdb.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-baa09e3f6fdb.json new file mode 100644 index 00000000..e21e13fb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-baa09e3f6fdb.json @@ -0,0 +1,50 @@ +{ + "id" : "1f8968cc-f8aa-3ef8-bb6e-38f1d42e01d1", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"input\" : [ {\n \"type\" : \"message\",\n \"role\" : \"system\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"Reply with one word.\"\n } ]\n }, {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"What is the capital of France?\"\n } ]\n } ],\n \"stream\" : false,\n \"store\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-baa09e3f6fdb.json", + "headers" : { + "x-request-id" : "req_c56257721db34539958fa3b24cb434ba", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a4b2a0639987c586-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "4ms", + "x-ratelimit-remaining-tokens" : "149999917", + "x-ratelimit-remaining-requests" : "29998", + "Date" : "Thu, 15 Oct 2026 23:33:03 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=EwAkkIukZUd4VPUPdanRC3hxFbcs9GYvzHLCljHnodI-1792107182.65643-1.0.1.1-MJJjwaFf5xoBDD9pV.mzdJB1UzAisRulE44YsjX3UQ68psL.tmi17YR5TRHPyaRZm61jE7XLv5LP6FTDGGv6zoO3uEMfyar.AJKMwnCaxrR8G6TkRbxjoBkor_Zp1f5v; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 16 Oct 2026 00:03:03 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "1122", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "1f8968cc-f8aa-3ef8-bb6e-38f1d42e01d1", + "persistent" : true, + "scenarioName" : "scenario-1-responses", + "requiredScenarioState" : "scenario-1-responses-2", + "insertionIndex" : 72 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-c2751be9bd77.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-c2751be9bd77.json new file mode 100644 index 00000000..94f5009b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-c2751be9bd77.json @@ -0,0 +1,48 @@ +{ + "id" : "1060ad4b-6b53-35b3-bed8-d12378ded8db", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the capital of France?\"}]}],\"model\":\"gpt-4o-mini\",\"instructions\":\"Reply with one word.\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-c2751be9bd77.json", + "headers" : { + "x-request-id" : "req_c4556d8d76b042c0ade7473ec63cc7ac", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a4b2a064daf9495d-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999960", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Thu, 15 Oct 2026 23:33:03 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=PMFVEryTmZ6BAWedgWfnbpkuTLL1BW9AlNzoShlOer0-1792107182.8542302-1.0.1.1-F5ekHiH6gKjV8PSvJ3D7oRBgZCiJ1s8PIujZeRewGKe79EKugLQ68CvNtW32raAOgb1pUzAsdzZpHrd6kaliC66dHV.trPq.vDelBzrm6qz_o6f2Mngelih5HzUFbw4I; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 16 Oct 2026 00:03:03 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "957", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "1060ad4b-6b53-35b3-bed8-d12378ded8db", + "persistent" : true, + "insertionIndex" : 71 +} \ No newline at end of file