diff --git a/e2e/helpers/anthropic-scenario.mjs b/e2e/helpers/anthropic-scenario.mjs index 80aa4ebf..9db07888 100644 --- a/e2e/helpers/anthropic-scenario.mjs +++ b/e2e/helpers/anthropic-scenario.mjs @@ -6,6 +6,20 @@ import { } from "./provider-runtime.mjs"; const ANTHROPIC_MODEL = "claude-3-haiku-20240307"; +const WEATHER_TOOL = { + name: "get_weather", + description: "Get the current weather in a given location", + input_schema: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state or city and country", + }, + }, + required: ["location"], + }, +}; export async function runAnthropicScenario(options) { const imageBase64 = (await readFile(options.testImageUrl)).toString("base64"); @@ -109,27 +123,39 @@ export async function runAnthropicScenario(options) { }, ); + await runOperation( + "anthropic-stream-tool-operation", + "stream-tool", + async () => { + const stream = await client.messages.create({ + model: ANTHROPIC_MODEL, + max_tokens: 128, + temperature: 0, + stream: true, + tool_choice: { + type: "tool", + name: WEATHER_TOOL.name, + disable_parallel_tool_use: true, + }, + tools: [WEATHER_TOOL], + messages: [ + { + role: "user", + content: + "Use the get_weather tool for Paris, France. Do not answer from memory.", + }, + ], + }); + await collectAsync(stream); + }, + ); + await runOperation("anthropic-tool-operation", "tool", async () => { await client.messages.create({ model: ANTHROPIC_MODEL, max_tokens: 128, temperature: 0, - tools: [ - { - name: "get_weather", - description: "Get the current weather in a given location", - input_schema: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state or city and country", - }, - }, - required: ["location"], - }, - }, - ], + tools: [WEATHER_TOOL], messages: [ { role: "user", diff --git a/e2e/helpers/anthropic-trace-contract.ts b/e2e/helpers/anthropic-trace-contract.ts index ef7f2729..9dd8ef0c 100644 --- a/e2e/helpers/anthropic-trace-contract.ts +++ b/e2e/helpers/anthropic-trace-contract.ts @@ -1,14 +1,8 @@ import { expect } from "vitest"; import { normalizeForSnapshot, type Json } from "./normalize"; -import type { - CapturedLogEvent, - CapturedLogPayload, -} from "./mock-braintrust-server"; +import type { CapturedLogEvent } from "./mock-braintrust-server"; import { findChildSpans, findLatestSpan } from "./trace-selectors"; -import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "./wrapper-contract"; +import { summarizeWrapperContract } from "./wrapper-contract"; function findNamedChildSpan( capturedEvents: CapturedLogEvent[], @@ -25,31 +19,65 @@ function findNamedChildSpan( return undefined; } -function normalizeAnthropicPayloads(payloadRows: unknown[]): unknown[] { - const attachmentRowKeys = new Set(); +function pickMetadata( + metadata: Record | undefined, + keys: string[], +): Json { + if (!metadata) { + return null; + } - for (const payload of payloadRows) { - if (!payload || typeof payload !== "object") { - continue; - } + const picked = Object.fromEntries( + keys.flatMap((key) => + key in metadata ? [[key, metadata[key] as Json]] : [], + ), + ); - const row = payload as { - id?: string; - input?: Array<{ - content?: - | string - | Array<{ - source?: { - data?: { - type?: string; - }; - }; - }>; - }>; - span_id?: string; - }; + return Object.keys(picked).length > 0 ? (picked as Json) : null; +} + +function summarizeAnthropicPayloadEvent( + event: CapturedLogEvent, + metadataKeys: string[], +): Json { + const summary = { + input: event.input as Json, + metadata: pickMetadata( + event.row.metadata as Record | undefined, + metadataKeys, + ), + metrics: event.metrics as Json, + name: event.span.name ?? null, + output: event.output as Json, + type: event.span.type ?? null, + } satisfies Json; - const hasAttachmentInput = row.input?.some( + if ( + event.span.name === "anthropic.messages.create" && + Array.isArray((summary.output as { content?: unknown[] } | null)?.content) + ) { + const output = structuredClone( + summary.output as { + content: Array<{ text?: string; type?: string }>; + }, + ); + const textBlock = output.content.find( + (block) => block.type === "text" && typeof block.text === "string", + ); + const input = event.input as + | Array<{ + content?: + | string + | Array<{ + source?: { + data?: { + type?: string; + }; + }; + }>; + }> + | undefined; + const hasAttachmentInput = input?.some( (message) => Array.isArray(message.content) && message.content.some( @@ -57,52 +85,17 @@ function normalizeAnthropicPayloads(payloadRows: unknown[]): unknown[] { ), ); - if (!hasAttachmentInput) { - continue; - } - - if (typeof row.id === "string") { - attachmentRowKeys.add(row.id); - } - if (typeof row.span_id === "string") { - attachmentRowKeys.add(row.span_id); + if (hasAttachmentInput && textBlock) { + textBlock.text = ""; + summary.output = output as Json; } } - return payloadRows.map((payload) => { - if (!payload || typeof payload !== "object") { - return payload; - } - - const row = structuredClone(payload) as { - id?: string; - metadata?: { operation?: string }; - output?: { - content?: Array<{ text?: string; type?: string }>; - }; - span_id?: string; - }; - const isAttachmentRow = - row.metadata?.operation === "attachment" || - (typeof row.id === "string" && attachmentRowKeys.has(row.id)) || - (typeof row.span_id === "string" && attachmentRowKeys.has(row.span_id)); - - if (isAttachmentRow) { - const textBlock = row.output?.content?.find( - (block) => block.type === "text" && typeof block.text === "string", - ); - if (textBlock) { - textBlock.text = ""; - } - } - - return row; - }); + return summary; } export function assertAnthropicTraceContract(options: { capturedEvents: CapturedLogEvent[]; - payloads: CapturedLogPayload[]; rootName: string; scenarioName: string; }): { payloadSummary: Json; spanSummary: Json } { @@ -119,6 +112,10 @@ export function assertAnthropicTraceContract(options: { options.capturedEvents, "anthropic-stream-with-response-operation", ); + const toolStreamOperation = findLatestSpan( + options.capturedEvents, + "anthropic-stream-tool-operation", + ); const toolOperation = findLatestSpan( options.capturedEvents, "anthropic-tool-operation", @@ -140,6 +137,7 @@ export function assertAnthropicTraceContract(options: { expect(createOperation).toBeDefined(); expect(streamOperation).toBeDefined(); expect(withResponseOperation).toBeDefined(); + expect(toolStreamOperation).toBeDefined(); expect(toolOperation).toBeDefined(); expect(attachmentOperation).toBeDefined(); expect(betaCreateOperation).toBeDefined(); @@ -151,6 +149,7 @@ export function assertAnthropicTraceContract(options: { expect(createOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(streamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(withResponseOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(toolStreamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(toolOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(attachmentOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(betaCreateOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); @@ -176,6 +175,11 @@ export function assertAnthropicTraceContract(options: { ["anthropic.messages.create"], toolOperation?.span.id, ); + const toolStreamSpan = findNamedChildSpan( + options.capturedEvents, + ["anthropic.messages.create"], + toolStreamOperation?.span.id, + ); const attachmentSpan = findNamedChildSpan( options.capturedEvents, ["anthropic.messages.create"], @@ -196,6 +200,7 @@ export function assertAnthropicTraceContract(options: { createSpan, streamSpan, withResponseSpan, + toolStreamSpan, toolSpan, attachmentSpan, betaCreateSpan, @@ -211,7 +216,12 @@ export function assertAnthropicTraceContract(options: { ).toBe("string"); } - for (const streamingSpan of [streamSpan, withResponseSpan, betaStreamSpan]) { + for (const streamingSpan of [ + streamSpan, + withResponseSpan, + toolStreamSpan, + betaStreamSpan, + ]) { expect(streamingSpan?.metrics).toMatchObject({ time_to_first_token: expect.any(Number), prompt_tokens: expect.any(Number), @@ -229,6 +239,16 @@ export function assertAnthropicTraceContract(options: { (block) => block.type === "tool_use" && block.name === "get_weather", ), ).toBe(true); + const toolStreamOutput = toolStreamSpan?.output as + | { + content?: Array<{ name?: string; type?: string }>; + } + | undefined; + expect( + toolStreamOutput?.content?.some( + (block) => block.type === "tool_use" && block.name === "get_weather", + ), + ).toBe(true); const attachmentInput = JSON.stringify(attachmentSpan?.input); expect(attachmentInput).toContain("image.png"); @@ -245,6 +265,8 @@ export function assertAnthropicTraceContract(options: { streamSpan, withResponseOperation, withResponseSpan, + toolStreamOperation, + toolStreamSpan, toolOperation, toolSpan, betaCreateOperation, @@ -261,8 +283,33 @@ export function assertAnthropicTraceContract(options: { ) as Json, ), payloadSummary: normalizeForSnapshot( - normalizeAnthropicPayloads( - payloadRowsForRootSpan(options.payloads, root?.span.id), + [ + root, + createOperation, + createSpan, + attachmentOperation, + attachmentSpan, + streamOperation, + streamSpan, + withResponseOperation, + withResponseSpan, + toolStreamOperation, + toolStreamSpan, + toolOperation, + toolSpan, + betaCreateOperation, + betaCreateSpan, + betaStreamOperation, + betaStreamSpan, + ].map((event) => + summarizeAnthropicPayloadEvent(event!, [ + "provider", + "model", + "operation", + "scenario", + "stop_reason", + "stop_sequence", + ]), ) as Json, ), }; diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json deleted file mode 100644 index 71bb1bc4..00000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json +++ /dev/null @@ -1,978 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "anthropic-auto-instrumentation-node-hook", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "anthropic-auto-hook-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "anthropic-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "time_to_first_token": 0, - "tokens": 17 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "OK.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "attachment", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "anthropic-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 29, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "time_to_first_token": 0, - "tokens": 1418 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "anthropic-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-with-response", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "anthropic-stream-with-response-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "anthropic-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 128, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0, - "tools": [ - { - "description": "Get the current weather in a given location", - "input_schema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - }, - "name": "get_weather" - } - ] - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 55, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 357, - "time_to_first_token": 0, - "tokens": 412 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "tool_use", - "stop_sequence": null - }, - "output": { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "anthropic-beta-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "anthropic.beta.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "time_to_first_token": 0, - "tokens": 19 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "BETA.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 13, - "name": "anthropic-beta-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 14, - "name": "anthropic.beta.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json deleted file mode 100644 index 6b468d32..00000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json +++ /dev/null @@ -1,281 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-auto-instrumentation-node-hook" - }, - "metric_keys": [], - "name": "anthropic-auto-hook-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-3-haiku-20240307", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs index 43e01c4f..1bb770e6 100644 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs +++ b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs @@ -6,8 +6,8 @@ runMain(async () => runAnthropicScenario({ Anthropic, projectNameBase: "e2e-anthropic-auto-instrumentation-hook", - rootName: "anthropic-auto-hook-root", - scenarioName: "anthropic-auto-instrumentation-node-hook", + rootName: "anthropic-wrapper-root", + scenarioName: "wrap-anthropic-message-traces", testImageUrl: new URL("./test-image.png", import.meta.url), useMessagesStreamHelper: false, }), diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts index 15a80302..42b4c2a7 100644 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts +++ b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts @@ -15,40 +15,45 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const sharedScenarioTestUrl = new URL( + "../wrap-anthropic-message-traces/scenario.test.ts", + import.meta.url, +).href; +const sharedSpanSnapshotPath = resolveFileSnapshotPath( + sharedScenarioTestUrl, + "span-events.json", +); +const sharedPayloadSnapshotPath = resolveFileSnapshotPath( + sharedScenarioTestUrl, + "log-payloads.json", +); test( - "anthropic auto-instrumentation via node hook collects the shared anthropic trace contract", + "anthropic auto-instrumentation via node hook matches the shared wrapper trace contract", { tags: [E2E_TAGS.externalApi], timeout: TIMEOUT_MS, }, async () => { - await withScenarioHarness( - async ({ events, payloads, runNodeScenarioDir }) => { - await runNodeScenarioDir({ - nodeArgs: ["--import", "braintrust/hook.mjs"], - scenarioDir, - timeoutMs: TIMEOUT_MS, - }); + await withScenarioHarness(async ({ events, runNodeScenarioDir }) => { + await runNodeScenarioDir({ + nodeArgs: ["--import", "braintrust/hook.mjs"], + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); - const contract = assertAnthropicTraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "anthropic-auto-hook-root", - scenarioName: "anthropic-auto-instrumentation-node-hook", - }); + const contract = assertAnthropicTraceContract({ + capturedEvents: events(), + rootName: "anthropic-wrapper-root", + scenarioName: "wrap-anthropic-message-traces", + }); - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }, - ); + await expect( + formatJsonFileSnapshot(contract.spanSummary), + ).toMatchFileSnapshot(sharedSpanSnapshotPath); + await expect( + formatJsonFileSnapshot(contract.payloadSummary), + ).toMatchFileSnapshot(sharedPayloadSnapshotPath); + }); }, ); diff --git a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json index 7e444e02..eb3fb2ec 100644 --- a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json +++ b/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json @@ -1,147 +1,50 @@ [ { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "scenario": "wrap-anthropic-message-traces", - "testRunId": "" + "scenario": "wrap-anthropic-message-traces" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "anthropic-wrapper-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" + "name": "anthropic-wrapper-root", + "type": "task" }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "create", - "testRunId": "" + "operation": "create" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "anthropic-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-create-operation", + "type": null }, { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Reply with exactly OK.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 16, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "anthropic.messages.create", - "type": "llm" + "stop_reason": "end_turn", + "stop_sequence": null }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 5, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, + "start": 0, "time_to_first_token": 0, "tokens": 17 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, + "name": "anthropic.messages.create", "output": { "content": [ { @@ -151,78 +54,20 @@ ], "role": "assistant" }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "attachment", - "testRunId": "" + "operation": "attachment" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "anthropic-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-attachment-operation", + "type": null }, { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": [ @@ -235,7 +80,7 @@ "data": { "content_type": "image/png", "filename": "image.png", - "key": "", + "key": "", "type": "braintrust_attachment" }, "media_type": "image/png", @@ -247,55 +92,23 @@ "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 32, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "anthropic.messages.create", - "type": "llm" + "stop_reason": "end_turn", + "stop_sequence": null }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 29, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, + "start": 0, "time_to_first_token": 0, "tokens": 1418 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, + "name": "anthropic.messages.create", "output": { "content": [ { @@ -305,510 +118,179 @@ ], "role": "assistant" }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "stream", - "testRunId": "" + "operation": "stream" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "anthropic-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-stream-operation", + "type": null }, { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Count from 1 to 3 and include the words one two three.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 32, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 + "stop_reason": "end_turn", + "stop_sequence": null }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 18, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, + "start": 0, "time_to_first_token": 0, "tokens": 42 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": null, - "stop_sequence": null - }, - "output": { - "content": [], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", + "name": "anthropic.messages.create", "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" }, { - "_is_merge": true, - "id": "", - "log_id": "g", "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-with-response", - "testRunId": "" + "operation": "stream-with-response" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "anthropic-stream-with-response-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-stream-with-response-operation", + "type": null }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/scenarios/wrap-anthropic-message-traces/node_modules/.pnpm/@anthropic-ai+sdk@/node_modules/@anthropic-ai/sdk/lib/MessageStream.js", - "caller_functionname": "MessageStream._createMessage", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Count from 1 to 3 and include the words one two three.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 32, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 + "stop_reason": "end_turn", + "stop_sequence": null }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 18, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, + "start": 0, "time_to_first_token": 0, "tokens": 42 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" }, { - "_is_merge": true, - "id": "", - "log_id": "g", "metadata": { - "stop_reason": null, - "stop_sequence": null + "operation": "stream-tool" }, - "output": { - "content": [], - "role": "assistant" + "metrics": { + "end": 0, + "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-stream-tool-operation", + "type": null }, { - "_is_merge": true, - "id": "", - "log_id": "g", + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], "metadata": { - "stop_reason": "end_turn", + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", "stop_sequence": null }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { - "end": 0 + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "tool", - "testRunId": "" + "operation": "tool" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "anthropic-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-tool-operation", + "type": null }, { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 128, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "temperature": 0, - "tools": [ - { - "description": "Get the current weather in a given location", - "input_schema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - }, - "name": "get_weather" - } - ] + "stop_reason": "tool_use", + "stop_sequence": null }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 55, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 357, + "start": 0, "time_to_first_token": 0, "tokens": 412 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "tool_use", - "stop_sequence": null - }, + "name": "anthropic.messages.create", "output": { "content": [ { "caller": { "type": "direct" }, - "id": "", + "id": "", "input": { "location": "Paris, France" }, @@ -818,133 +300,43 @@ ], "role": "assistant" }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" }, { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "beta-create", - "testRunId": "" + "operation": "beta-create" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "anthropic-beta-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-beta-create-operation", + "type": null }, { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Reply with exactly BETA.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 16, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "anthropic.messages.create", - "type": "llm" + "stop_reason": "end_turn", + "stop_sequence": null }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 6, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, + "start": 0, "time_to_first_token": 0, "tokens": 19 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, + "name": "anthropic.messages.create", "output": { "content": [ { @@ -954,201 +346,44 @@ ], "role": "assistant" }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" }, { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", "metadata": { - "operation": "beta-stream", - "testRunId": "" + "operation": "beta-stream" }, "metrics": { + "end": 0, "start": 0 }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 13, - "name": "anthropic-beta-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] + "name": "anthropic-beta-stream-operation", + "type": null }, { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", "input": [ { "content": "Count from 1 to 3 and include the words one two three.", "role": "user" } ], - "log_id": "g", "metadata": { - "max_tokens": 32, "model": "claude-3-haiku-20240307", "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 14, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 + "stop_reason": "end_turn", + "stop_sequence": null }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", "metrics": { "completion_tokens": 18, + "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, + "start": 0, "time_to_first_token": 0, "tokens": 42 }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": null, - "stop_sequence": null - }, - "output": { - "content": [], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", + "name": "anthropic.messages.create", "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] + "type": "llm" } ] diff --git a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json b/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json index 7d054c87..d4fef738 100644 --- a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json +++ b/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json @@ -168,10 +168,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "tool" + "operation": "stream-tool" }, "metric_keys": [], - "name": "anthropic-tool-operation", + "name": "anthropic-stream-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -206,10 +206,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-create" + "operation": "tool" }, "metric_keys": [], - "name": "anthropic-beta-create-operation", + "name": "anthropic-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -244,10 +244,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-stream" + "operation": "beta-create" }, "metric_keys": [], - "name": "anthropic-beta-stream-operation", + "name": "anthropic-beta-create-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -277,5 +277,43 @@ "" ], "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" } ] diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.impl.mjs b/e2e/scenarios/wrap-anthropic-message-traces/scenario.impl.mjs new file mode 100644 index 00000000..b7886182 --- /dev/null +++ b/e2e/scenarios/wrap-anthropic-message-traces/scenario.impl.mjs @@ -0,0 +1,27 @@ +import { wrapAnthropic } from "braintrust"; +import { runAnthropicScenario } from "../../helpers/anthropic-scenario.mjs"; + +const ROOT_NAME = "anthropic-wrapper-root"; +const SCENARIO_NAME = "wrap-anthropic-message-traces"; + +export async function runWrapAnthropicMessageTraces(Anthropic) { + await runAnthropicScenario({ + Anthropic, + decorateClient: wrapAnthropic, + projectNameBase: "e2e-wrap-anthropic", + rootName: ROOT_NAME, + scenarioName: SCENARIO_NAME, + testImageUrl: new URL("./test-image.png", import.meta.url), + }); +} + +export async function runAnthropicAutoInstrumentationNodeHook(Anthropic) { + await runAnthropicScenario({ + Anthropic, + projectNameBase: "e2e-anthropic-auto-instrumentation-hook", + rootName: ROOT_NAME, + scenarioName: SCENARIO_NAME, + testImageUrl: new URL("./test-image.png", import.meta.url), + useMessagesStreamHelper: false, + }); +} diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.mjs b/e2e/scenarios/wrap-anthropic-message-traces/scenario.mjs new file mode 100644 index 00000000..7b25b46e --- /dev/null +++ b/e2e/scenarios/wrap-anthropic-message-traces/scenario.mjs @@ -0,0 +1,5 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAnthropicAutoInstrumentationNodeHook } from "./scenario.impl.mjs"; + +runMain(async () => runAnthropicAutoInstrumentationNodeHook(Anthropic)); diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts b/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts index a435c2d3..1b034183 100644 --- a/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts +++ b/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts @@ -15,6 +15,73 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const sharedSpanSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "span-events.json", +); +const sharedPayloadSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "log-payloads.json", +); + +type AnthropicContract = ReturnType; + +let wrapperContractPromise: Promise | undefined; +let autoContractPromise: Promise | undefined; + +function getWrapperContract(): Promise { + wrapperContractPromise ??= (async () => { + let contract: AnthropicContract | undefined; + + await withScenarioHarness(async ({ events, runScenarioDir }) => { + await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); + + contract = assertAnthropicTraceContract({ + capturedEvents: events(), + rootName: "anthropic-wrapper-root", + scenarioName: "wrap-anthropic-message-traces", + }); + }); + + if (!contract) { + throw new Error("Failed to capture Anthropic wrapper contract"); + } + + return contract; + })(); + + return wrapperContractPromise; +} + +function getAutoContract(): Promise { + autoContractPromise ??= (async () => { + let contract: AnthropicContract | undefined; + + await withScenarioHarness(async ({ events, runNodeScenarioDir }) => { + await runNodeScenarioDir({ + nodeArgs: ["--import", "braintrust/hook.mjs"], + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + + contract = assertAnthropicTraceContract({ + capturedEvents: events(), + rootName: "anthropic-wrapper-root", + scenarioName: "wrap-anthropic-message-traces", + }); + }); + + if (!contract) { + throw new Error( + "Failed to capture Anthropic auto-instrumentation contract", + ); + } + + return contract; + })(); + + return autoContractPromise; +} test( "wrap-anthropic-message-traces captures create, stream, beta, attachment, and tool spans", @@ -23,26 +90,37 @@ test( timeout: TIMEOUT_MS, }, async () => { - await withScenarioHarness(async ({ events, payloads, runScenarioDir }) => { - await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); + const contract = await getWrapperContract(); - const contract = assertAnthropicTraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "anthropic-wrapper-root", - scenarioName: "wrap-anthropic-message-traces", - }); + await expect( + formatJsonFileSnapshot(contract.spanSummary), + ).toMatchFileSnapshot(sharedSpanSnapshotPath); + await expect( + formatJsonFileSnapshot(contract.payloadSummary), + ).toMatchFileSnapshot(sharedPayloadSnapshotPath); + }, +); - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }); +test( + "anthropic auto-instrumentation via node hook matches the wrapper trace contract", + { + tags: [E2E_TAGS.externalApi], + timeout: TIMEOUT_MS, + }, + async () => { + const [wrapperContract, autoContract] = await Promise.all([ + getWrapperContract(), + getAutoContract(), + ]); + + expect(autoContract.payloadSummary).toEqual(wrapperContract.payloadSummary); + expect(autoContract.spanSummary).toEqual(wrapperContract.spanSummary); + + await expect( + formatJsonFileSnapshot(autoContract.spanSummary), + ).toMatchFileSnapshot(sharedSpanSnapshotPath); + await expect( + formatJsonFileSnapshot(autoContract.payloadSummary), + ).toMatchFileSnapshot(sharedPayloadSnapshotPath); }, ); diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts b/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts index 95fa46e0..4b321712 100644 --- a/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts +++ b/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts @@ -1,15 +1,5 @@ import Anthropic from "@anthropic-ai/sdk"; -import { wrapAnthropic } from "braintrust"; -import { runAnthropicScenario } from "../../helpers/anthropic-scenario.mjs"; import { runMain } from "../../helpers/scenario-runtime"; +import { runWrapAnthropicMessageTraces } from "./scenario.impl.mjs"; -runMain(async () => - runAnthropicScenario({ - Anthropic, - decorateClient: wrapAnthropic, - projectNameBase: "e2e-wrap-anthropic", - rootName: "anthropic-wrapper-root", - scenarioName: "wrap-anthropic-message-traces", - testImageUrl: new URL("./test-image.png", import.meta.url), - }), -); +runMain(async () => runWrapAnthropicMessageTraces(Anthropic)); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index 19a8f2e5..889f83d0 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -385,6 +385,77 @@ describe("aggregateAnthropicStreamChunks", () => { expect(result.output).toBe("Hello world"); }); + it("should aggregate streamed tool_use output", () => { + const chunks = [ + { + type: "message_start", + message: { + role: "assistant", + usage: { + input_tokens: 25, + output_tokens: 0, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: "toolu_123", + name: "get_weather", + input: {}, + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "input_json_delta", + partial_json: '{"location":"Paris, France"}', + }, + }, + { + type: "content_block_stop", + index: 0, + }, + { + type: "message_delta", + delta: { + stop_reason: "tool_use", + stop_sequence: null, + }, + usage: { + output_tokens: 12, + }, + }, + ]; + + const result = aggregateAnthropicStreamChunks(chunks); + + expect(result.output).toEqual({ + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_123", + name: "get_weather", + input: { + location: "Paris, France", + }, + }, + ], + }); + expect(result.metrics).toMatchObject({ + prompt_tokens: 25, + completion_tokens: 12, + }); + expect(result.metadata).toEqual({ + stop_reason: "tool_use", + stop_sequence: null, + }); + }); + it("should handle chunks without delta or usage", () => { const chunks = [ { type: "message_start" }, diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index f0dbbf15..5bb62161 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -10,6 +10,7 @@ import type { AnthropicCreateParams, AnthropicInputMessage, AnthropicMessage, + AnthropicOutputContentBlock, AnthropicStreamEvent, AnthropicUsage, } from "../../vendor-sdk-types/anthropic"; @@ -91,7 +92,7 @@ export class AnthropicPlugin extends BasePlugin { this.unsubscribers.push( traceStreamingChannel(anthropicChannels.betaMessagesCreate, { ...anthropicConfig, - name: "anthropic.beta.messages.create", + name: "anthropic.messages.create", }), ); } @@ -138,13 +139,16 @@ export function parseMetricsFromUsage( export function aggregateAnthropicStreamChunks( chunks: AnthropicStreamEvent[], ): { - output: string; + output: unknown; metrics: Record; metadata: Record; } { - const deltas: string[] = []; + const fallbackTextDeltas: string[] = []; + const contentBlocks: Record = {}; + const contentBlockDeltas: Record = {}; let metrics: Record = {}; let metadata: Record = {}; + let role: string | undefined; for (const event of chunks) { switch (event?.type) { @@ -154,18 +158,50 @@ export function aggregateAnthropicStreamChunks( const initialMetrics = parseMetricsFromUsage(event.message.usage); metrics = { ...metrics, ...initialMetrics }; } + if (typeof event.message?.role === "string") { + role = event.message.role; + } + break; + + case "content_block_start": + if (event.content_block) { + contentBlocks[event.index] = event.content_block; + contentBlockDeltas[event.index] = []; + } break; case "content_block_delta": - // Collect text deltas if (event.delta?.type === "text_delta") { const text = event.delta.text; if (text) { - deltas.push(text); + if ( + contentBlocks[event.index] !== undefined || + contentBlockDeltas[event.index] !== undefined + ) { + contentBlockDeltas[event.index] ??= []; + contentBlockDeltas[event.index].push(text); + } else { + fallbackTextDeltas.push(text); + } + } + } else if (event.delta?.type === "input_json_delta") { + const partialJson = event.delta.partial_json; + if (partialJson) { + contentBlockDeltas[event.index] ??= []; + contentBlockDeltas[event.index].push(partialJson); } } break; + case "content_block_stop": + finalizeContentBlock( + event.index, + contentBlocks, + contentBlockDeltas, + fallbackTextDeltas, + ); + break; + case "message_delta": // Collect final usage stats and metadata if (event.usage) { @@ -180,7 +216,26 @@ export function aggregateAnthropicStreamChunks( } } - const output = deltas.join(""); + const orderedContent = Object.entries(contentBlocks) + .map(([index, block]) => ({ + block, + index: Number(index), + })) + .filter(({ block }) => block !== undefined) + .sort((left, right) => left.index - right.index) + .map(({ block }) => block); + + let output: unknown = fallbackTextDeltas.join(""); + if (orderedContent.length > 0) { + if (orderedContent.every(isTextContentBlock)) { + output = orderedContent.map((block) => block.text).join(""); + } else { + output = { + ...(role ? { role } : {}), + content: orderedContent, + }; + } + } const finalized = finalizeAnthropicTokens(metrics); // Filter out undefined values to match Record type @@ -197,6 +252,67 @@ export function aggregateAnthropicStreamChunks( }; } +function finalizeContentBlock( + index: number, + contentBlocks: Record, + contentBlockDeltas: Record, + fallbackTextDeltas: string[], +): void { + const contentBlock = contentBlocks[index]; + if (!contentBlock) { + return; + } + + const text = contentBlockDeltas[index]?.join("") ?? ""; + + if (isToolUseContentBlock(contentBlock)) { + if (!text) { + return; + } + + try { + contentBlocks[index] = { + ...contentBlock, + input: JSON.parse(text), + }; + } catch { + fallbackTextDeltas.push(text); + delete contentBlocks[index]; + } + return; + } + + if (isTextContentBlock(contentBlock)) { + if (!text) { + delete contentBlocks[index]; + return; + } + + contentBlocks[index] = { + ...contentBlock, + text, + }; + return; + } + + if (text) { + fallbackTextDeltas.push(text); + } + delete contentBlocks[index]; +} + +function isTextContentBlock( + contentBlock: AnthropicOutputContentBlock, +): contentBlock is Extract { + return contentBlock.type === "text"; +} + +function isToolUseContentBlock( + contentBlock: AnthropicOutputContentBlock, +): contentBlock is Extract { + return contentBlock.type === "tool_use"; +} + function isAnthropicBase64ContentBlock( input: Record, ): input is Record & {