merge final reviewed results stack

apps/cli/src/commands/eval/artifact-writer.ts

@@ -94,6 +94,7 @@ export function buildIndexArtifactEntry(
     outputPath?: string;
     answerPath?: string;
     transcriptPath?: string;
+    rawProviderLogPath?: string;
     inputPath?: string;
     responsePath?: string;
     taskBundle?: MaterializedTaskBundlePaths;

apps/cli/src/commands/eval/run-eval.ts

@@ -361,6 +361,16 @@ export function trimOutputMessages(
   return output;
 }
 
+export function prepareResultForJsonl(
+  result: EvaluationResult,
+  options: { readonly outputMessages: number | 'all' },
+): EvaluationResult {
+  return {
+    ...result,
+    output: trimOutputMessages(result.output, options.outputMessages),
+  };
+}
+
 function normalizeOptions(
   rawOptions: Record<string, unknown>,
   config?: Awaited<ReturnType<typeof loadTsConfig>>,
@@ -1043,11 +1053,7 @@ async function runSingleEvalFile(params: {
       // Each message is trimmed to { role, content } only (no toolCalls, startTime, etc.).
       // Full output with tool calls goes to OTel.
       const resultWithMetadata = withSourceMetadata(result, testFilePath, options);
-      const trimmedOutput = trimOutputMessages(resultWithMetadata.output, options.outputMessages);
-      const trimmedResult: EvaluationResult = {
-        ...resultWithMetadata,
-        output: trimmedOutput,
-      };
+      const trimmedResult = prepareResultForJsonl(resultWithMetadata, options);
       await outputWriter.append(trimmedResult);
 
       // Export to OTel if exporter is configured (skip batch export when streaming is active)

apps/cli/src/commands/inspect/utils.ts

@@ -8,7 +8,7 @@ import {
   resolveExistingRunPrimaryPath,
   resolveWorkspaceOrFilePath,
 } from '../eval/result-layout.js';
-import { loadManifestResults } from '../results/manifest.js';
+import { loadLightweightResults, loadManifestResults } from '../results/manifest.js';
 import { ResultRowSchemaError, normalizeResultRow } from '../results/result-row-schema.js';
 
 // ANSI color codes (no dependency needed)
@@ -636,7 +636,7 @@ export function listResultFilesFromRunsDir(runsDir: string, limit?: number): Res
   for (const { filePath, displayName, runId } of limited) {
     try {
       const fileStat = statSync(filePath);
-      const results = loadResultFile(filePath);
+      const results = loadLightweightResults(filePath);
 
       const testCount = results.length;
       const passCount = results.filter((r) => r.score >= DEFAULT_THRESHOLD).length;

apps/cli/src/commands/results/combine-run.ts

@@ -20,7 +20,12 @@ import {
 } from 'node:fs';
 import path from 'node:path';
 
-import type { EvaluationResult } from '@agentv/core';
+import type {
+  EvaluationResult,
+  ResultArtifactPointerWire,
+  ResultArtifactPointersWire,
+  TranscriptArtifactPointerWire,
+} from '@agentv/core';
 
 import {
   type BenchmarkArtifact,
@@ -305,26 +310,38 @@ function toRunId(cwd: string, runDir: string): string {
 }
 
 const MANIFEST_PATH_FIELDS = [
+  'artifact_dir',
   'grading_path',
   'timing_path',
   'input_path',
   'output_path',
   'response_path',
+  'transcript_path',
+  'raw_provider_log_path',
   'task_dir',
   'eval_path',
   'targets_path',
   'files_path',
   'graders_path',
 ] as const;
 
+const POINTER_FAMILIES = {
+  trace: 'traces',
+  transcript: 'transcripts',
+} as const;
+
+function isSafeRelativeArtifactPath(relativePath: string): boolean {
+  return !path.isAbsolute(relativePath) && !relativePath.split(/[\\/]+/).includes('..');
+}
+
 function copyReferencedArtifact(
   sourceBaseDir: string,
   outputDir: string,
   sourceIndex: number,
   relativePath: string | undefined,
 ): string | undefined {
   if (!relativePath) return undefined;
-  if (path.isAbsolute(relativePath) || relativePath.split(/[\\/]+/).includes('..')) {
+  if (!isSafeRelativeArtifactPath(relativePath)) {
     throw new Error(`Unsafe artifact path in source manifest: ${relativePath}`);
   }
   const sourcePath = path.join(sourceBaseDir, relativePath);
@@ -343,6 +360,71 @@ function copyReferencedArtifact(
   return rewritten;
 }
 
+function rewriteArtifactPointer(
+  pointerName: keyof typeof POINTER_FAMILIES,
+  pointer: ResultArtifactPointerWire | undefined,
+  sourceBaseDir: string,
+  outputDir: string,
+  sourceIndex: number,
+): ResultArtifactPointerWire | undefined {
+  if (!pointer) {
+    return undefined;
+  }
+
+  if (!isSafeRelativeArtifactPath(pointer.path)) {
+    throw new Error(`Unsafe artifact path in source manifest: ${pointer.path}`);
+  }
+  const sourcePath = path.join(sourceBaseDir, pointer.path);
+  if (!existsSync(sourcePath)) {
+    return { ...pointer };
+  }
+
+  const rewrittenPath = copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, pointer.path);
+  if (!rewrittenPath) {
+    return { ...pointer };
+  }
+
+  const family = pointer.family ?? POINTER_FAMILIES[pointerName];
+  return {
+    ...pointer,
+    path: rewrittenPath,
+    key: path.posix.join(family, rewrittenPath),
+  };
+}
+
+function rewriteTranscriptArtifactPointer(
+  pointer: TranscriptArtifactPointerWire | undefined,
+  sourceBaseDir: string,
+  outputDir: string,
+  sourceIndex: number,
+): TranscriptArtifactPointerWire | undefined {
+  return rewriteArtifactPointer('transcript', pointer, sourceBaseDir, outputDir, sourceIndex) as
+    | TranscriptArtifactPointerWire
+    | undefined;
+}
+
+function rewriteArtifactPointers(
+  pointers: ResultArtifactPointersWire | undefined,
+  sourceBaseDir: string,
+  outputDir: string,
+  sourceIndex: number,
+): ResultArtifactPointersWire | undefined {
+  if (!pointers) {
+    return undefined;
+  }
+
+  return {
+    ...pointers,
+    trace: rewriteArtifactPointer('trace', pointers.trace, sourceBaseDir, outputDir, sourceIndex),
+    transcript: rewriteTranscriptArtifactPointer(
+      pointers.transcript,
+      sourceBaseDir,
+      outputDir,
+      sourceIndex,
+    ),
+  };
+}
+
 function rewriteAndCopyRecord(row: SelectedRow, outputDir: string): ResultManifestRecord {
   const sourceBaseDir = path.dirname(row.source.manifestPath);
   const rewritten: Record<string, unknown> = { ...row.record };
@@ -354,6 +436,20 @@ function rewriteAndCopyRecord(row: SelectedRow, outputDir: string): ResultManife
       row.record[field],
     );
   }
+  const artifactPointers = rewriteArtifactPointers(
+    row.record.artifact_pointers,
+    sourceBaseDir,
+    outputDir,
+    row.source.index,
+  );
+  rewritten.artifact_pointers = artifactPointers;
+  if (
+    row.record.transcript_path &&
+    rewritten.transcript_path === row.record.transcript_path &&
+    artifactPointers?.transcript?.path
+  ) {
+    rewritten.transcript_path = artifactPointers.transcript.path;
+  }
   return rewritten as unknown as ResultManifestRecord;
 }
 

apps/cli/src/commands/results/export.ts

@@ -21,14 +21,22 @@
  *   - To add new per-test workspace files, add them under each test directory.
  */
 
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 
-import { command, oneOf, option, optional, positional, string } from 'cmd-ts';
+import { command, flag, oneOf, option, optional, positional, string } from 'cmd-ts';
 
-import type { EvaluationResult, ExportDuplicatePolicy } from '@agentv/core';
+import type { EvaluationResult, ExportDuplicatePolicy, IndexArtifactEntry } from '@agentv/core';
 
 import { parseJsonlResults, writeArtifactsFromResults } from '../eval/artifact-writer.js';
 import { RESULT_INDEX_FILENAME } from '../eval/result-layout.js';
+import { loadManifestResults } from './manifest.js';
+import {
+  type ProjectionBundle,
+  buildProjectionBundle,
+  serializeProjectionBundle,
+  writeProjectionBundle,
+} from './projection-bundle.js';
 import { loadResults as loadSharedResults, resolveSourceFile } from './shared.js';
 
 // ── Export logic ─────────────────────────────────────────────────────────
@@ -92,6 +100,36 @@ export async function loadExportSource(
   return { sourceFile, results };
 }
 
+function readIndexArtifactEntries(indexPath: string): IndexArtifactEntry[] {
+  return readFileSync(indexPath, 'utf8')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as IndexArtifactEntry);
+}
+
+export function buildProjectionBundleFromExportedIndex(options: {
+  readonly sourceFile: string;
+  readonly outputDir: string;
+  readonly cwd?: string;
+  readonly includeRawContent?: boolean;
+  readonly duplicatePolicy?: ExportDuplicatePolicy;
+}): ProjectionBundle {
+  const indexPath = path.join(options.outputDir, RESULT_INDEX_FILENAME);
+  const indexRecords = readIndexArtifactEntries(indexPath);
+  const emittedResults = loadManifestResults(indexPath);
+
+  return buildProjectionBundle(emittedResults, {
+    sourceFile: options.sourceFile,
+    runId: deriveExportRunId(options.sourceFile),
+    cwd: options.cwd,
+    duplicatePolicy: options.duplicatePolicy,
+    includeRawContent: options.includeRawContent,
+    artifactRefStatus: 'emitted',
+    indexRecords,
+  });
+}
+
 // ── CLI command ──────────────────────────────────────────────────────────
 
 export const resultsExportCommand = command({
@@ -122,10 +160,34 @@ export const resultsExportCommand = command({
       description:
         'How to handle duplicate projection identities in the output: update (default), skip, or error',
     }),
+    projectionBundle: flag({
+      long: 'projection-bundle',
+      description: 'Write a vendor-neutral projection_bundle.json alongside exported artifacts',
+    }),
+    dryRun: flag({
+      long: 'dry-run',
+      description: 'Print deterministic projection bundle JSON without writing export artifacts',
+    }),
+    includeRawContent: flag({
+      long: 'include-raw-content',
+      description:
+        'Include raw prompt, output, and tool payload content in the projection bundle (off by default)',
+    }),
   },
-  handler: async ({ source, out, dir, duplicatePolicy }) => {
+  handler: async ({
+    source,
+    out,
+    dir,
+    duplicatePolicy,
+    projectionBundle,
+    dryRun,
+    includeRawContent,
+  }) => {
     const cwd = dir ?? process.cwd();
     const policy = (duplicatePolicy ?? 'update') as ExportDuplicatePolicy;
+    const shouldWriteProjectionBundle = projectionBundle;
+    const shouldDryRun = dryRun;
+    const shouldIncludeRawContent = includeRawContent;
 
     try {
       const { sourceFile, results } = await loadExportSource(source, cwd);
@@ -136,14 +198,44 @@ export const resultsExportCommand = command({
           : path.resolve(cwd, out)
         : deriveOutputDir(cwd, sourceFile);
 
+      const buildBundle = () =>
+        buildProjectionBundle(results, {
+          sourceFile,
+          runId: deriveExportRunId(sourceFile),
+          cwd,
+          duplicatePolicy: policy,
+          includeRawContent: shouldIncludeRawContent,
+        });
+
+      if (shouldDryRun) {
+        process.stdout.write(serializeProjectionBundle(buildBundle()));
+        return;
+      }
+
       await writeArtifactsFromResults(results, outputDir, {
         evalFile: sourceFile,
         runId: deriveExportRunId(sourceFile),
         duplicatePolicy: policy,
       });
 
+      const bundlePath = shouldWriteProjectionBundle
+        ? await writeProjectionBundle(
+            buildProjectionBundleFromExportedIndex({
+              sourceFile,
+              outputDir,
+              cwd,
+              duplicatePolicy: policy,
+              includeRawContent: shouldIncludeRawContent,
+            }),
+            outputDir,
+          )
+        : undefined;
+
       // Report exported test IDs
       console.log(`Exported ${results.length} test(s) to ${outputDir}`);
+      if (bundlePath) {
+        console.log(`Projection bundle written to ${bundlePath}`);
+      }
       for (const result of results) {
         console.log(`  ${result.testId ?? 'unknown'}`);
       }