Agent Experiment API Reference

ExperimentConfig.builder()
    .experimentName("name")                    // Required: experiment identifier
    .datasetDir(Path.of("dataset"))            // Required: dataset directory
    .model("sonnet")                           // Required: LLM model
    .promptTemplate("{{task}}")                // Required: prompt with placeholders
    .perItemTimeout(Duration.ofMinutes(2))     // Required: per-item timeout
    .itemFilter(ItemFilter.bucket("A"))        // Optional: filter items
    .knowledgeBaseDir(Path.of("kb"))           // Optional: KB root
    .outputDir(Path.of("results"))             // Optional: persist workspaces
    .experimentTimeout(Duration.ofHours(1))    // Optional: overall timeout
    .metadata(Map.of("key", "value"))          // Optional: arbitrary metadata
    .baselineId("abc123")                      // Optional: comparison baseline
    .build();

public interface AgentInvoker {
    InvocationResult invoke(InvocationContext context)
        throws AgentInvocationException;
}

// Success
InvocationResult.completed(phases, inputTokens, outputTokens,
    thinkingTokens, totalCostUsd, durationMs, sessionId, metadata);

// Timeout
InvocationResult.timeout(durationMs, metadata, errorMessage);

// Error
InvocationResult.error(errorMessage, metadata);

public interface ExecutionDetail {
    // Marker — shared infrastructure stores but never interprets
}

if (item.executionDetail() instanceof InvocationResult inv) {
    System.out.println("Cost: $" + inv.totalCostUsd());
}

{
  "schemaVersion": 1,
  "name": "dataset-name",
  "version": "1.0.0",
  "description": "What this dataset tests",
  "items": [
    {
      "id": "ITEM-001",
      "slug": "short-description",
      "path": "items/ITEM-001",
      "bucket": "A",
      "taskType": "task-type",
      "status": "active"
    }
  ]
}

{
  "schemaVersion": 1,
  "id": "ITEM-001",
  "slug": "short-description",
  "developerTask": "Natural language task description",
  "taskType": "task-type",
  "bucket": "A",
  "noChange": false,
  "knowledgeRefs": ["path/to/kb-entry.md"],
  "tags": ["tag1", "tag2"],
  "status": "active"
}

dataset/
├── dataset.json
└── items/
    └── ITEM-001/
        ├── item.json
        ├── before/          # Starting state
        │   └── src/...
        └── reference/       # Correct result
            └── src/...

ItemFilter.all()                     // No filtering
ItemFilter.bucket("A")               // Single bucket
ItemFilter.tags("rename", "simple")  // By tags
ItemFilter.id("ITEM-001")           // Single item

void save(ExperimentResult result);
Optional<ExperimentResult> load(String id);
List<ExperimentResult> listByName(String experimentName);
Optional<ExperimentResult> mostRecent(String experimentName);

@FunctionalInterface
public interface ReEvaluationContextFactory {
    Optional<JudgmentContext> create(ItemResult item);
}

ReEvaluationContextFactory factory =
    AgentReEvaluationContextFactory.defaults();

ReEvaluator reEvaluator = ReEvaluator.builder()
    .contextFactory(AgentReEvaluationContextFactory.defaults())
    .resultStore(store)
    .build();

ExperimentResult reScored = reEvaluator.reEvaluate(originalResult, newJury);

@FunctionalInterface
public interface JudgeScorer {
    JudgeScorerResult score(JudgeScoringInput input);
}

public record JudgeScoringInput(
    DatasetItem item,        // dataset item (for item-level context)
    Judgment actual,         // candidate judge's judgment
    String expectedLabel     // expected label from dataset
)

public record JudgeScorerResult(
    boolean match,           // judge agreed with expected label
    double score,            // normalized agreement score [0, 1]
    String reasoning         // explanation of match/mismatch
)

public record JudgeExecutionDetail(
    Judgment candidateJudgment,
    String expectedLabel,
    JudgeScorerResult scorerResult
) implements ExecutionDetail

JudgeExperimentResult result = JudgeExperiment.builder()
    .name("correctness-judge-calibration")
    .candidate(myCorrectnessJudge)
    .items(labeledItems)
    .input(item -> buildContextFromItem(item))
    .expected(item -> item.metadata().get("expectedVerdict"))
    .scorer(JudgeScorers.exactVerdictMatch())
    .resultStore(store)
    .build()
    .run();

public record JudgeExperimentResult(
    ExperimentResult experimentResult,
    double agreementRate,
    List<JudgeDisagreement> disagreements
)

public record JudgeDisagreement(
    String itemId,
    JudgeExecutionDetail detail
)

<!-- Core: runner, dataset, jury, stores, diagnostics -->
<dependency>
    <groupId>io.github.markpollack</groupId>
    <artifactId>experiment-core</artifactId>
</dependency>

<!-- Claude SDK integration (ClaudeSdkInvoker, SemanticDiffJudge) -->
<dependency>
    <groupId>io.github.markpollack</groupId>
    <artifactId>experiment-claude</artifactId>
</dependency>