Agent Experiment API Reference

ExperimentConfig.builder()
    .experimentName("name")                    // Required: experiment identifier
    .datasetDir(Path.of("dataset"))            // Required: dataset directory
    .model("sonnet")                           // Required: LLM model
    .promptTemplate("{{task}}")                // Required: prompt with placeholders
    .perItemTimeout(Duration.ofMinutes(2))     // Required: per-item timeout
    .itemFilter(ItemFilter.bucket("A"))        // Optional: filter items
    .knowledgeBaseDir(Path.of("kb"))           // Optional: KB root
    .outputDir(Path.of("results"))             // Optional: persist workspaces
    .experimentTimeout(Duration.ofHours(1))    // Optional: overall timeout
    .metadata(Map.of("key", "value"))          // Optional: arbitrary metadata
    .baselineId("abc123")                      // Optional: comparison baseline
    .build();

public interface AgentInvoker {
    InvocationResult invoke(InvocationContext context)
        throws AgentInvocationException;
}

// Success
InvocationResult.completed(phases, inputTokens, outputTokens,
    thinkingTokens, totalCostUsd, durationMs, sessionId, metadata);

// Timeout
InvocationResult.timeout(durationMs, metadata, errorMessage);

// Error
InvocationResult.error(errorMessage, metadata);

{
  "schemaVersion": 1,
  "name": "dataset-name",
  "version": "1.0.0",
  "description": "What this dataset tests",
  "items": [
    {
      "id": "ITEM-001",
      "slug": "short-description",
      "path": "items/ITEM-001",
      "bucket": "A",
      "taskType": "task-type",
      "status": "active"
    }
  ]
}

{
  "schemaVersion": 1,
  "id": "ITEM-001",
  "slug": "short-description",
  "developerTask": "Natural language task description",
  "taskType": "task-type",
  "bucket": "A",
  "noChange": false,
  "knowledgeRefs": ["path/to/kb-entry.md"],
  "tags": ["tag1", "tag2"],
  "status": "active"
}

dataset/
├── dataset.json
└── items/
    └── ITEM-001/
        ├── item.json
        ├── before/          # Starting state
        │   └── src/...
        └── reference/       # Correct result
            └── src/...

ItemFilter.all()                     // No filtering
ItemFilter.bucket("A")               // Single bucket
ItemFilter.tags("rename", "simple")  // By tags
ItemFilter.id("ITEM-001")           // Single item

void save(ExperimentResult result);
Optional<ExperimentResult> load(String id);
List<ExperimentResult> listByName(String experimentName);
Optional<ExperimentResult> mostRecent(String experimentName);

<!-- Core: runner, dataset, jury, stores, diagnostics -->
<dependency>
    <groupId>ai.tuvium</groupId>
    <artifactId>experiment-core</artifactId>
</dependency>

<!-- Claude SDK integration (ClaudeSdkInvoker, SemanticDiffJudge) -->
<dependency>
    <groupId>ai.tuvium</groupId>
    <artifactId>experiment-claude</artifactId>
</dependency>