diff --git a/command-snapshot.json b/command-snapshot.json index 3c8eb216..7ce251d6 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -257,6 +257,14 @@ "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": ["d", "f", "s", "t"], + "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:validate:authoring-bundle", diff --git a/messages/agent.trace.read.md b/messages/agent.trace.read.md new file mode 100644 index 00000000..f6fb1de3 --- /dev/null +++ b/messages/agent.trace.read.md @@ -0,0 +1,171 @@ +# summary + +Read and analyze trace files from an agent preview session. + +# description + +Reads trace files recorded during an agent preview session and outputs them in one of three formats. + +**--format summary** (default): A per-turn narrative showing topic routing, actions executed, and the agent's response. Use this to quickly understand what happened in a session. + +**--format detail**: Diagnostic drill-down into a specific dimension (--dimension required). Filters output to only the trace steps relevant to that dimension, minimizing noise. + +**--format raw**: Unprocessed trace JSON. Use this as a fallback when the trace schema has changed or you need to perform custom analysis. + +Available dimensions for --format detail: actions, grounding, routing, errors. + +Use --turn N to scope output to a single conversation turn. + +# flags.session-id.summary + +Session ID to read traces for. + +# flags.format.summary + +Output format: summary (default), detail, or raw. Use detail with --dimension to drill into a specific aspect of the trace. + +# flags.dimension.summary + +Dimension to drill into when using --format detail. One of: actions, grounding, routing, errors. Required when --format is detail. + +# flags.turn.summary + +Scope output to this conversation turn number. + +# error.detailRequiresDimension + +--format detail requires --dimension. Specify one of: actions, grounding, routing, errors. + +# error.sessionNotFound + +Session '%s' was not found in the local session cache. Run "sf agent trace list" to see available sessions. + +# error.turnIndexNotFound + +No turn index found for session '%s'. Cannot filter by --turn without a turn index. + +# error.turnNotFound + +Turn %s was not found in session '%s'. + +# error.parseFailedAll + +Trace parsing failed for all files: %s. The trace schema may have changed. Try --format raw to access unprocessed trace data. + +# warn.dimensionIgnored + +--dimension is ignored when --format is '%s'. Use --format detail to drill into a dimension. + +# warn.parseFailed + +Trace parsing failed for some files (skipped): %s. Try --format raw to access unprocessed trace data. + +# output.empty + +No traces found for this session. + +# output.emptyDimension + +No '%s' data found in the traces for this session. + +# output.tableHeader.turn + +Turn + +# output.tableHeader.topic + +Topic + +# output.tableHeader.userInput + +User Input + +# output.tableHeader.agentResponse + +Agent Response + +# output.tableHeader.actionsExecuted + +Actions Executed + +# output.tableHeader.latencyMs + +Latency + +# output.tableHeader.error + +Error + +# output.tableHeader.action + +Action + +# output.tableHeader.input + +Input + +# output.tableHeader.output + +Output + +# output.tableHeader.prompt + +Prompt + +# output.tableHeader.response + +Response + +# output.tableHeader.intent + +Intent + +# output.tableHeader.fromTopic + +From Topic + +# output.tableHeader.toTopic + +To Topic + +# output.tableHeader.source + +Source + +# output.tableHeader.errorCode + +Error Code + +# output.tableHeader.message + +Message + +# examples + +- Show a session summary (all turns): + + <%= config.bin %> <%= command.id %> --session-id + +- Show summary for a single turn: + + <%= config.bin %> <%= command.id %> --session-id --turn 2 + +- Drill into action execution across all turns: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension actions + +- Drill into routing decisions for a specific turn: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension routing --turn 1 + +- Show all errors across the session: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension errors + +- Output raw trace JSON for custom parsing: + + <%= config.bin %> <%= command.id %> --session-id --format raw + +- Return results as JSON: + + <%= config.bin %> <%= command.id %> --session-id --json diff --git a/schemas/agent-trace-read.json b/schemas/agent-trace-read.json new file mode 100644 index 00000000..e8162963 --- /dev/null +++ b/schemas/agent-trace-read.json @@ -0,0 +1,466 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTraceReadResult", + "definitions": { + "AgentTraceReadResult": { + "type": "object", + "properties": { + "sessionId": { + "type": "string" + }, + "format": { + "type": "string", + "enum": ["summary", "detail", "raw"] + }, + "dimension": { + "$ref": "#/definitions/Dimension" + }, + "turns": { + "type": "array", + "items": { + "$ref": "#/definitions/TurnSummary" + } + }, + "detail": { + "type": "array", + "items": { + "$ref": "#/definitions/DimensionRow" + } + }, + "raw": { + "type": "array", + "items": { + "$ref": "#/definitions/PlannerResponse" + } + } + }, + "required": ["sessionId", "format"], + "additionalProperties": false + }, + "Dimension": { + "type": "string", + "enum": ["actions", "grounding", "routing", "errors"] + }, + "TurnSummary": { + "type": "object", + "properties": { + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "userInput": { + "type": "string" + }, + "agentResponse": { + "type": "string" + }, + "actionsExecuted": { + "type": "array", + "items": { + "type": "string" + } + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["turn", "planId", "topic", "userInput", "agentResponse", "actionsExecuted", "latencyMs", "error"], + "additionalProperties": false + }, + "DimensionRow": { + "anyOf": [ + { + "$ref": "#/definitions/ActionsRow" + }, + { + "$ref": "#/definitions/GroundingRow" + }, + { + "$ref": "#/definitions/RoutingRow" + }, + { + "$ref": "#/definitions/ErrorsRow" + } + ] + }, + "ActionsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "actions" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "action": { + "type": "string" + }, + "input": { + "type": "string" + }, + "output": { + "type": "string" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["dimension", "turn", "planId", "action", "input", "output", "latencyMs", "error"], + "additionalProperties": false + }, + "GroundingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "grounding" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "response": { + "type": "string" + }, + "latencyMs": { + "type": "number" + } + }, + "required": ["dimension", "turn", "planId", "prompt", "response", "latencyMs"], + "additionalProperties": false + }, + "RoutingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "routing" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "fromTopic": { + "type": "string" + }, + "toTopic": { + "type": "string" + }, + "intent": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "fromTopic", "toTopic", "intent"], + "additionalProperties": false + }, + "ErrorsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "errors" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "source": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "source", "errorCode", "message"], + "additionalProperties": false + }, + "PlannerResponse": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlanSuccessResponse" + }, + "planId": { + "type": "string" + }, + "sessionId": { + "type": "string" + }, + "intent": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "plan": { + "type": "array", + "items": { + "$ref": "#/definitions/PlanStep" + } + } + }, + "required": ["type", "planId", "sessionId", "intent", "topic", "plan"], + "additionalProperties": false + }, + "PlanStep": { + "anyOf": [ + { + "$ref": "#/definitions/UserInputStep" + }, + { + "$ref": "#/definitions/LLMExecutionStep" + }, + { + "$ref": "#/definitions/UpdateTopicStep" + }, + { + "$ref": "#/definitions/EventStep" + }, + { + "$ref": "#/definitions/FunctionStep" + }, + { + "$ref": "#/definitions/PlannerResponseStep" + } + ] + }, + "UserInputStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UserInputStep" + }, + "message": { + "type": "string" + } + }, + "required": ["type", "message"], + "additionalProperties": false + }, + "LLMExecutionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "LLMExecutionStep" + }, + "promptName": { + "type": "string" + }, + "promptContent": { + "type": "string" + }, + "promptResponse": { + "type": "string" + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": [ + "type", + "promptName", + "promptContent", + "promptResponse", + "executionLatency", + "startExecutionTime", + "endExecutionTime" + ], + "additionalProperties": false + }, + "UpdateTopicStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UpdateTopicStep" + }, + "topic": { + "type": "string" + }, + "description": { + "type": "string" + }, + "job": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": { + "type": "string" + } + }, + "availableFunctions": { + "type": "array", + "items": {} + } + }, + "required": ["type", "topic", "description", "job", "instructions", "availableFunctions"], + "additionalProperties": false + }, + "EventStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "EventStep" + }, + "eventName": { + "type": "string" + }, + "isError": { + "type": "boolean" + }, + "payload": { + "type": "object", + "properties": { + "oldTopic": { + "type": "string" + }, + "newTopic": { + "type": "string" + } + }, + "required": ["oldTopic", "newTopic"], + "additionalProperties": false + } + }, + "required": ["type", "eventName", "isError", "payload"], + "additionalProperties": false + }, + "FunctionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "FunctionStep" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "input": { + "type": "object", + "additionalProperties": {} + }, + "output": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["name", "input", "output"], + "additionalProperties": false + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": ["type", "function", "executionLatency", "startExecutionTime", "endExecutionTime"], + "additionalProperties": false + }, + "PlannerResponseStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlannerResponseStep" + }, + "message": { + "type": "string" + }, + "responseType": { + "type": "string" + }, + "isContentSafe": { + "type": "boolean" + }, + "safetyScore": { + "type": "object", + "properties": { + "safety_score": { + "type": "number" + }, + "category_scores": { + "type": "object", + "properties": { + "toxicity": { + "type": "number" + }, + "hate": { + "type": "number" + }, + "identity": { + "type": "number" + }, + "violence": { + "type": "number" + }, + "physical": { + "type": "number" + }, + "sexual": { + "type": "number" + }, + "profanity": { + "type": "number" + }, + "biased": { + "type": "number" + } + }, + "required": ["toxicity", "hate", "identity", "violence", "physical", "sexual", "profanity", "biased"], + "additionalProperties": false + } + }, + "required": ["safety_score", "category_scores"], + "additionalProperties": false + } + }, + "required": ["type", "message", "responseType", "isContentSafe", "safetyScore"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/trace/read.ts b/src/commands/agent/trace/read.ts new file mode 100644 index 00000000..d3011b71 --- /dev/null +++ b/src/commands/agent/trace/read.ts @@ -0,0 +1,429 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages, SfError } from '@salesforce/core'; +import { + listCachedPreviewSessions, + listSessionTraces, + readSessionTrace, + readTurnIndex, + type PlannerResponse, + type PlanStep, + type FunctionStep, +} from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.trace.read'); + +export const DIMENSIONS = ['actions', 'grounding', 'routing', 'errors'] as const; +export type Dimension = (typeof DIMENSIONS)[number]; + +// FunctionStep in @salesforce/agents doesn't declare the optional errors field that the API returns +type FunctionStepWithErrors = FunctionStep & { + function: FunctionStep['function'] & { + errors?: Array<{ statusCode: string; message: string }>; + }; +}; + +export type TurnSummary = { + turn: number; + planId: string; + topic: string; + userInput: string; + agentResponse: string; + actionsExecuted: string[]; + latencyMs: number; + error: string | null; +}; + +export type ActionsRow = { + dimension: 'actions'; + turn: number; + planId: string; + action: string; + input: string; + output: string; + latencyMs: number; + error: string | null; +}; +export type GroundingRow = { + dimension: 'grounding'; + turn: number; + planId: string; + prompt: string; + response: string; + latencyMs: number; +}; +export type RoutingRow = { + dimension: 'routing'; + turn: number; + planId: string; + fromTopic: string; + toTopic: string; + intent: string; +}; +export type ErrorsRow = { + dimension: 'errors'; + turn: number; + planId: string; + source: string; + errorCode: string; + message: string; +}; +export type DimensionRow = ActionsRow | GroundingRow | RoutingRow | ErrorsRow; + +export type AgentTraceReadResult = { + sessionId: string; + format: 'summary' | 'detail' | 'raw'; + dimension?: Dimension; + turns?: TurnSummary[]; + detail?: DimensionRow[]; + raw?: PlannerResponse[]; +}; + +const isFunctionStep = (s: PlanStep): s is FunctionStep => s.type === 'FunctionStep'; +const asFunctionWithErrors = (s: FunctionStep): FunctionStepWithErrors => s as FunctionStepWithErrors; + +function summarizeTurn(turn: number, planId: string, trace: PlannerResponse): TurnSummary { + const plan = trace.plan; + const userInput = plan.find((s) => s.type === 'UserInputStep'); + const finalResponse = plan.find((s) => s.type === 'PlannerResponseStep'); + const functionSteps = plan.filter(isFunctionStep).map(asFunctionWithErrors); + + const errorStep = functionSteps.find((s) => s.function.errors?.length); + const errorMsg = errorStep?.function.errors?.[0]?.message ?? null; + const totalLatency = functionSteps.reduce((acc, s) => acc + (s.executionLatency ?? 0), 0); + + return { + turn, + planId, + topic: trace.topic, + userInput: userInput?.type === 'UserInputStep' ? userInput.message : '', + agentResponse: finalResponse?.type === 'PlannerResponseStep' ? finalResponse.message : '', + actionsExecuted: functionSteps.map((s) => s.function.name), + latencyMs: totalLatency, + error: errorMsg, + }; +} + +function extractActions(turn: number, planId: string, trace: PlannerResponse): ActionsRow[] { + return trace.plan + .filter(isFunctionStep) + .map(asFunctionWithErrors) + .map((step) => ({ + dimension: 'actions' as const, + turn, + planId, + action: step.function.name, + input: JSON.stringify(step.function.input), + output: JSON.stringify(step.function.output), + latencyMs: step.executionLatency, + error: step.function.errors?.length ? step.function.errors[0].message : null, + })); +} + +function extractGrounding(turn: number, planId: string, trace: PlannerResponse): GroundingRow[] { + return trace.plan + .filter((s): s is Extract => s.type === 'LLMExecutionStep') + .filter((s) => s.promptName.includes('React')) + .map((step) => ({ + dimension: 'grounding' as const, + turn, + planId, + prompt: step.promptName, + response: step.promptResponse.slice(0, 500), + latencyMs: step.executionLatency, + })); +} + +function extractRouting(turn: number, planId: string, trace: PlannerResponse): RoutingRow[] { + const topicStep = trace.plan.find((s) => s.type === 'UpdateTopicStep'); + const eventStep = trace.plan.find((s) => s.type === 'EventStep' && s.eventName === 'topicChangeEvent'); + const fromTopic = eventStep?.type === 'EventStep' ? eventStep.payload.oldTopic : 'null'; + const toTopic = topicStep?.type === 'UpdateTopicStep' ? topicStep.topic : trace.topic; + return [{ dimension: 'routing' as const, turn, planId, fromTopic, toTopic, intent: trace.intent }]; +} + +function extractErrors(turn: number, planId: string, trace: PlannerResponse): ErrorsRow[] { + const rows: ErrorsRow[] = []; + for (const step of trace.plan) { + if (step.type === 'FunctionStep') { + const errors = asFunctionWithErrors(step).function.errors ?? []; + for (const e of errors) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.function.name, + errorCode: e.statusCode, + message: e.message, + }); + } + } + if (step.type === 'EventStep' && step.isError) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.eventName, + errorCode: 'EVENT_ERROR', + message: JSON.stringify(step.payload), + }); + } + } + return rows; +} + +async function resolvePlanIds( + agentId: string, + sessionId: string, + turn: number | undefined +): Promise> { + const turnIndex = await readTurnIndex(agentId, sessionId); + + if (turn !== undefined) { + if (!turnIndex) { + throw new SfError(messages.getMessage('error.turnIndexNotFound', [sessionId]), 'TurnIndexNotFound'); + } + const entry = turnIndex.turns.find((t) => t.turn === turn && t.planId); + if (!entry?.planId) { + throw new SfError(messages.getMessage('error.turnNotFound', [turn, sessionId]), 'TurnNotFound'); + } + return [{ turn: entry.turn, planId: entry.planId }]; + } + + if (turnIndex) { + return turnIndex.turns.filter((t) => t.planId !== null).map((t) => ({ turn: t.turn, planId: t.planId! })); + } + + // Fall back to listing trace files when no turn index exists + const traceFiles = await listSessionTraces(agentId, sessionId); + return traceFiles.map((f, i) => ({ turn: i + 1, planId: f.planId })); +} + +async function readTraces( + agentId: string, + sessionId: string, + planIds: Array<{ turn: number; planId: string }> +): Promise<{ traces: Array<{ turn: number; planId: string; trace: PlannerResponse }>; failedFiles: string[] }> { + const traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> = []; + const failedFiles: string[] = []; + + for (const { turn, planId } of planIds) { + // eslint-disable-next-line no-await-in-loop + const trace = await readSessionTrace(agentId, sessionId, planId); + if (!trace?.plan || !Array.isArray(trace.plan)) { + failedFiles.push(planId); + continue; + } + traces.push({ turn, planId, trace }); + } + + return { traces, failedFiles }; +} + +function extractDimension(turn: number, planId: string, trace: PlannerResponse, dimension: Dimension): DimensionRow[] { + if (dimension === 'actions') return extractActions(turn, planId, trace); + if (dimension === 'grounding') return extractGrounding(turn, planId, trace); + if (dimension === 'routing') return extractRouting(turn, planId, trace); + return extractErrors(turn, planId, trace); +} + +export default class AgentTraceRead extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly flags = { + 'session-id': Flags.string({ + summary: messages.getMessage('flags.session-id.summary'), + required: true, + char: 's', + }), + format: Flags.option({ + options: ['summary', 'detail', 'raw'] as const, + default: 'summary' as const, + summary: messages.getMessage('flags.format.summary'), + char: 'f', + })(), + dimension: Flags.option({ + options: DIMENSIONS, + summary: messages.getMessage('flags.dimension.summary'), + char: 'd', + })(), + turn: Flags.integer({ + summary: messages.getMessage('flags.turn.summary'), + char: 't', + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTraceRead); + const sessionId = flags['session-id']; + + if (flags.format === 'detail' && !flags.dimension) { + throw new SfError(messages.getMessage('error.detailRequiresDimension'), 'MissingDimension'); + } + if (flags.dimension && flags.format !== 'detail') { + this.warn(messages.getMessage('warn.dimensionIgnored', [flags.format])); + } + + const agentId = await this.resolveAgentId(sessionId); + const planIds = await resolvePlanIds(agentId, sessionId, flags.turn); + + if (planIds.length === 0) { + this.log(messages.getMessage('output.empty')); + return { sessionId, format: flags.format, turns: [], detail: [], raw: [] }; + } + + const { traces, failedFiles } = await readTraces(agentId, sessionId, planIds); + + if (failedFiles.length > 0 && traces.length === 0) { + throw new SfError(messages.getMessage('error.parseFailedAll', [failedFiles.join(', ')]), 'TraceParseError'); + } + if (failedFiles.length > 0) { + this.warn(messages.getMessage('warn.parseFailed', [failedFiles.join(', ')])); + } + + return this.formatOutput(sessionId, flags.format, flags.dimension, traces); + } + + private async resolveAgentId(sessionId: string): Promise { + const cachedAgents = await listCachedPreviewSessions(this.project!); + const entry = cachedAgents.find((a) => a.sessions.some((s) => s.sessionId === sessionId)); + if (!entry) { + throw new SfError(messages.getMessage('error.sessionNotFound', [sessionId]), 'SessionNotFound'); + } + return entry.agentId; + } + + private formatOutput( + sessionId: string, + format: 'summary' | 'detail' | 'raw', + dimension: Dimension | undefined, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + if (format === 'raw') { + const raw = traces.map((t) => t.trace); + if (!this.jsonEnabled()) this.log(JSON.stringify(raw, null, 2)); + return { sessionId, format: 'raw', raw }; + } + + if (format === 'detail') { + return this.formatDetail(sessionId, dimension!, traces); + } + + return this.formatSummary(sessionId, traces); + } + + private formatDetail( + sessionId: string, + dimension: Dimension, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const detail: DimensionRow[] = traces.flatMap(({ turn, planId, trace }) => + extractDimension(turn, planId, trace, dimension) + ); + + if (detail.length === 0) { + this.log(messages.getMessage('output.emptyDimension', [dimension])); + return { sessionId, format: 'detail', dimension, detail: [] }; + } + + if (!this.jsonEnabled()) { + this.renderDetailTable(dimension, detail); + } + + return { sessionId, format: 'detail', dimension, detail }; + } + + private renderDetailTable(dimension: Dimension, detail: DimensionRow[]): void { + if (dimension === 'actions') { + this.table({ + data: detail as ActionsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'action', name: messages.getMessage('output.tableHeader.action') }, + { key: 'input', name: messages.getMessage('output.tableHeader.input') }, + { key: 'output', name: messages.getMessage('output.tableHeader.output') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } else if (dimension === 'grounding') { + this.table({ + data: detail as GroundingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'prompt', name: messages.getMessage('output.tableHeader.prompt') }, + { key: 'response', name: messages.getMessage('output.tableHeader.response') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + ], + }); + } else if (dimension === 'routing') { + this.table({ + data: detail as RoutingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'intent', name: messages.getMessage('output.tableHeader.intent') }, + { key: 'fromTopic', name: messages.getMessage('output.tableHeader.fromTopic') }, + { key: 'toTopic', name: messages.getMessage('output.tableHeader.toTopic') }, + ], + }); + } else { + this.table({ + data: detail as ErrorsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'source', name: messages.getMessage('output.tableHeader.source') }, + { key: 'errorCode', name: messages.getMessage('output.tableHeader.errorCode') }, + { key: 'message', name: messages.getMessage('output.tableHeader.message') }, + ], + }); + } + } + + private formatSummary( + sessionId: string, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const turns: TurnSummary[] = traces.map(({ turn, planId, trace }) => summarizeTurn(turn, planId, trace)); + + if (!this.jsonEnabled()) { + this.table({ + data: turns.map((t) => ({ + ...t, + actionsExecuted: t.actionsExecuted.join(', ') || '—', + error: t.error ?? '—', + latencyMs: `${t.latencyMs}ms`, + })), + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'topic', name: messages.getMessage('output.tableHeader.topic') }, + { key: 'userInput', name: messages.getMessage('output.tableHeader.userInput') }, + { key: 'agentResponse', name: messages.getMessage('output.tableHeader.agentResponse') }, + { key: 'actionsExecuted', name: messages.getMessage('output.tableHeader.actionsExecuted') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } + + return { sessionId, format: 'summary', turns }; + } +} diff --git a/test/commands/agent/trace/read.test.ts b/test/commands/agent/trace/read.test.ts new file mode 100644 index 00000000..4688308a --- /dev/null +++ b/test/commands/agent/trace/read.test.ts @@ -0,0 +1,462 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unnecessary-type-assertion */ + +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; + +const MOCK_PROJECT_DIR = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template'); + +const SESSION_ID = 'sess-abc'; +const AGENT_ID = 'AgentA'; +const PLAN_ID_1 = 'plan-1'; +const PLAN_ID_2 = 'plan-2'; + +const MOCK_CACHED_SESSIONS = [ + { + agentId: AGENT_ID, + displayName: 'My_Agent_A', + sessions: [{ sessionId: SESSION_ID, timestamp: '2026-04-07T17:00:00.000Z' }], + }, +]; + +const MOCK_TURN_INDEX = { + version: '1', + sessionId: SESSION_ID, + agentId: AGENT_ID, + created: '2026-04-07T17:00:00.000Z', + turns: [ + { + turn: 1, + timestamp: '2026-04-07T17:00:00.000Z', + role: 'user', + summary: 'Hi!', + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_1}.json`, + planId: PLAN_ID_1, + }, + { + turn: 2, + timestamp: '2026-04-07T17:00:01.000Z', + role: 'user', + summary: "what's the weather", + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_2}.json`, + planId: PLAN_ID_2, + }, + ], +}; + +// Simple off-topic trace (no actions, no errors) +const MOCK_TRACE_1 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_1, + sessionId: SESSION_ID, + intent: 'Off_Topic', + topic: 'Off_Topic', + plan: [ + { type: 'UserInputStep', message: 'Hi!' }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Off_Topic', + executionLatency: 460, + startExecutionTime: 1000, + endExecutionTime: 1460, + }, + { + type: 'UpdateTopicStep', + topic: 'Off_Topic', + description: 'Off topic', + job: 'redirect', + instructions: [], + availableFunctions: [], + }, + { + type: 'EventStep', + eventName: 'topicChangeEvent', + isError: false, + payload: { oldTopic: 'null', newTopic: 'Off_Topic' }, + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: 'Hey there!', + executionLatency: 1637, + startExecutionTime: 1461, + endExecutionTime: 3098, + }, + { + type: 'PlannerResponseStep', + message: 'Hey there! How can I assist you today?', + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +// Weather trace with action + error +const MOCK_TRACE_2 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_2, + sessionId: SESSION_ID, + intent: 'Local_Weather', + topic: 'Local_Weather', + plan: [ + { type: 'UserInputStep', message: "what's the weather" }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Local_Weather', + executionLatency: 572, + startExecutionTime: 2000, + endExecutionTime: 2572, + }, + { + type: 'UpdateTopicStep', + topic: 'Local_Weather', + description: 'Weather', + job: 'answer weather questions', + instructions: [], + availableFunctions: ['Check_Weather'], + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: + '- id: call_xxx\n function:\n name: Check_Weather\n arguments: \'{"dateToCheck":"2025-08-18"}\'', + executionLatency: 748, + startExecutionTime: 2600, + endExecutionTime: 3348, + }, + { + type: 'FunctionStep', + function: { + name: 'Check_Weather', + input: { dateToCheck: '2025-08-18' }, + output: {}, + errors: [{ statusCode: 'UNKNOWN_EXCEPTION', message: 'Bad response: 404' }], + }, + executionLatency: 781, + startExecutionTime: 3350, + endExecutionTime: 4131, + }, + { + type: 'PlannerResponseStep', + message: "I'm having trouble accessing the weather.", + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +describe('agent trace read', () => { + const $$ = new TestContext(); + let listCachedPreviewSessionsStub: sinon.SinonStub; + let listSessionTracesStub: sinon.SinonStub; + let readSessionTraceStub: sinon.SinonStub; + let readTurnIndexStub: sinon.SinonStub; + let AgentTraceRead: any; + + beforeEach(async () => { + listCachedPreviewSessionsStub = $$.SANDBOX.stub().resolves(MOCK_CACHED_SESSIONS); + listSessionTracesStub = $$.SANDBOX.stub().resolves([]); + readTurnIndexStub = $$.SANDBOX.stub().resolves(MOCK_TURN_INDEX); + readSessionTraceStub = $$.SANDBOX.stub(); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_1).resolves(MOCK_TRACE_1); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_2); + + const mod = await esmock('../../../../src/commands/agent/trace/read.js', { + '@salesforce/agents': { + listCachedPreviewSessions: listCachedPreviewSessionsStub, + listSessionTraces: listSessionTracesStub, + readSessionTrace: readSessionTraceStub, + readTurnIndex: readTurnIndexStub, + }, + }); + + AgentTraceRead = mod.default; + + $$.inProject(true); + const mockProject = { getPath: () => MOCK_PROJECT_DIR } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + }); + + describe('--format summary (default)', () => { + it('returns summary for all turns when no --turn specified', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.format).to.equal('summary'); + expect(result.turns).to.have.length(2); + }); + + it('each turn has required fields', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + const turn = result.turns[0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + }); + + it('populates topic and userInput from trace', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].topic).to.equal('Off_Topic'); + expect(result.turns[0].userInput).to.equal('Hi!'); + }); + + it('lists executed actions', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].actionsExecuted).to.deep.equal(['Check_Weather']); + }); + + it('captures error from failed function step', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].error).to.equal('Bad response: 404'); + }); + + it('error is null when no function errors exist', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].error).to.be.null; + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect(result.turns).to.have.length(1); + expect(result.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('throws when --dimension is missing', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'detail']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('--dimension'); + } + }); + + describe('--dimension actions', () => { + it('returns only action rows', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.format).to.equal('detail'); + expect(result.dimension).to.equal('actions'); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ action: 'Check_Weather' }); + }); + + it('includes error details in action row', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail![0].error).to.equal('Bad response: 404'); + }); + + it('returns empty when no actions exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension routing', () => { + it('returns routing rows for each turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + ]); + expect(result.detail).to.have.length(2); + expect(result.detail![0]).to.include({ fromTopic: 'null', toTopic: 'Off_Topic', intent: 'Off_Topic' }); + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + '--turn', + '2', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ intent: 'Local_Weather' }); + }); + }); + + describe('--dimension errors', () => { + it('returns rows only for turns with errors', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ source: 'Check_Weather', errorCode: 'UNKNOWN_EXCEPTION' }); + }); + + it('returns empty when no errors exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension grounding', () => { + it('returns LLM execution steps with React prompts', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'grounding', + ]); + expect(result.detail!.length).to.be.greaterThan(0); + for (const row of result.detail!) { + expect((row as any).prompt).to.include('React'); + } + }); + }); + }); + + describe('--format raw', () => { + it('returns raw trace objects', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.format).to.equal('raw'); + expect(result.raw).to.have.length(2); + expect(result.raw![0].planId).to.equal(PLAN_ID_1); + }); + + it('raw output matches the full trace structure', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.raw![0]).to.deep.equal(MOCK_TRACE_1); + }); + }); + + describe('validation and error handling', () => { + it('throws when session is not found', async () => { + listCachedPreviewSessionsStub.resolves([]); + try { + await AgentTraceRead.run(['--session-id', 'no-such-session']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no-such-session'); + } + }); + + it('throws when --turn is used but no turn index exists', async () => { + readTurnIndexStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn index/i); + } + }); + + it('throws when --turn number does not exist in the index', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '99']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn 99|not found/i); + } + }); + + it('throws when all trace files fail to parse', async () => { + readSessionTraceStub.resetBehavior(); + readSessionTraceStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/Trace parsing failed|raw/i); + } + }); + + it('returns empty result when session has no trace files and no turn index', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.deep.equal([]); + }); + + it('falls back to listSessionTraces when no turn index exists', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([{ planId: PLAN_ID_1, path: '/path/plan-1.json', size: 1000, mtime: new Date() }]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.have.length(1); + expect(result.turns![0].topic).to.equal('Off_Topic'); + }); + }); +}); diff --git a/test/nuts/z4.agent.trace.read.nut.ts b/test/nuts/z4.agent.trace.read.nut.ts new file mode 100644 index 00000000..2c6227f7 --- /dev/null +++ b/test/nuts/z4.agent.trace.read.nut.ts @@ -0,0 +1,177 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; +import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; +import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; +import type { AgentTraceReadResult } from '../../src/commands/agent/trace/read.js'; +import { getTestSession, getUsername } from './shared-setup.js'; + +describe('agent trace read', function () { + this.timeout(30 * 60 * 1000); + + let session: TestSession; + let sessionId: string; + const bundleApiName = 'Willie_Resort_Manager'; + + before(async function () { + this.timeout(30 * 60 * 1000); + session = await getTestSession(); + + // Start a preview session with two turns so there are traces to read + const targetOrg = getUsername(); + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + expect(startResult?.sessionId).to.be.a('string'); + sessionId = startResult!.sessionId; + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "What can you help me with?" --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "Tell me more" --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + + execCmd( + `agent preview end --session-id ${sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + }); + + describe('--format summary (default)', () => { + it('returns a summary result for the session', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('summary'); + expect(result?.sessionId).to.equal(sessionId); + expect(result?.turns).to.be.an('array').with.length.greaterThan(0); + }); + + it('each turn has the required summary fields', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + const turn = result!.turns![0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + expect(turn.userInput).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.agentResponse).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.topic).to.be.a('string').and.have.length.greaterThan(0); + }); + + it('scopes to a single turn with --turn 1', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --turn 1 --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.turns).to.have.length(1); + expect(result?.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('errors when --dimension is missing', () => { + execCmd(`agent trace read --session-id ${sessionId} --format detail --json`, { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + + it('returns routing dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension routing --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.dimension).to.equal('routing'); + expect(result?.detail).to.be.an('array').with.length.greaterThan(0); + const row = result!.detail![0] as { turn: number; intent: string; toTopic: string }; + expect(row.turn).to.be.a('number'); + expect(row.intent).to.be.a('string'); + expect(row.toTopic).to.be.a('string'); + }); + + it('returns grounding dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension grounding --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.detail).to.be.an('array').with.length.greaterThan(0); + const row = result!.detail![0] as { prompt: string; latencyMs: number }; + expect(row.prompt).to.be.a('string').and.include('React'); + expect(row.latencyMs).to.be.a('number').and.greaterThanOrEqual(0); + }); + + it('returns actions dimension rows (may be empty for off-topic sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension actions --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + + it('returns errors dimension rows (may be empty for successful sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension errors --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + }); + + describe('--format raw', () => { + it('returns raw trace JSON', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --format raw --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('raw'); + expect(result?.raw).to.be.an('array').with.length.greaterThan(0); + const trace = result!.raw![0] as { type: string; plan: unknown[] }; + expect(trace.type).to.equal('PlanSuccessResponse'); + expect(trace.plan).to.be.an('array').with.length.greaterThan(0); + }); + }); + + describe('error handling', () => { + it('errors when session is not found', () => { + execCmd('agent trace read --session-id no-such-session --json', { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + }); +});