Getting Started with Agent Experiment

# Clone the template
git clone https://github.com/markpollack/agent-experiment-template my-experiment
cd my-experiment

# Run the baseline variant
./mvnw compile exec:java -Dexec.args="--variant control"

# Run all variants and compare
./mvnw compile exec:java -Dexec.args="--run-all-variants"

my-dataset/
├── dataset.json
└── items/
    └── RENAME-001/
        ├── item.json
        ├── before/
        │   └── src/main/java/com/example/Person.java
        └── reference/
            └── src/main/java/com/example/Person.java

{
  "schemaVersion": 1,
  "name": "rename-field",
  "version": "1.0.0",
  "description": "Field rename tasks",
  "items": [
    {
      "id": "RENAME-001",
      "slug": "simple-rename",
      "path": "items/RENAME-001",
      "bucket": "A",
      "taskType": "rename-field",
      "status": "active"
    }
  ]
}

{
  "schemaVersion": 1,
  "id": "RENAME-001",
  "slug": "simple-rename",
  "developerTask": "Rename the field 'name' to 'fullName' in Person.java and update all references",
  "taskType": "rename-field",
  "bucket": "A",
  "noChange": false,
  "knowledgeRefs": [],
  "tags": ["rename", "simple"],
  "status": "active"
}

// WorkflowAgentInvoker works out of the box — journal wired, knowledge injected.
// Rename to {Domain}AgentInvoker and override hooks as needed.

public class MyWorkflow extends WorkflowInvoker<MyState> {

    @Override protected String workflowName() { return "my-experiment"; }

    @Override
    protected Workflow<Object, MyState> buildWorkflow(
            InvocationContext ctx, WorkflowExecutor executor) {
        return Workflow.<Object, MyState>define(workflowName())
                .withExecutor(executor)   // journal + cost tracking pre-wired
                .step(analyzeStep)
                .step(fixStep)
                .build();
    }

    @Override
    protected MyState buildInitialState(InvocationContext ctx) {
        return new MyState(ctx.workspacePath());
    }
}

public class MyAgent implements AgentInvoker {

    @Override
    public InvocationResult invoke(InvocationContext context) {
        // Your agent works in context.workspacePath()
        // using context.prompt() as the task description

        ProcessBuilder pb = new ProcessBuilder(
            "my-agent", "--workspace", context.workspacePath().toString(),
                        "--prompt", context.prompt());
        pb.directory(context.workspacePath().toFile());

        Process p = pb.start();
        boolean finished = p.waitFor(
            context.timeout().toSeconds(), TimeUnit.SECONDS);

        if (!finished) {
            p.destroyForcibly();
            return InvocationResult.timeout(
                context.timeout().toMillis(),
                context.metadata(), "Timed out");
        }

        return InvocationResult.completed(
            List.of(), 0, 0, 0, 0.0,
            System.currentTimeMillis(),
            null, context.metadata());
    }
}

public class FileExistsJudge implements Judge, JudgeWithMetadata {
    private final String expectedFile;

    public FileExistsJudge(String expectedFile) {
        this.expectedFile = expectedFile;
    }

    @Override
    public Judgment judge(JudgmentContext context) {
        boolean exists = Files.exists(
            context.workspacePath().resolve(expectedFile));

        return exists
            ? Judgment.pass("Found")
            : Judgment.fail("Missing: " + expectedFile);
    }

    @Override
    public JudgeMetadata metadata() {
        return new JudgeMetadata(
            "file_exists",
            "Checks that " + expectedFile + " exists",
            JudgeType.DETERMINISTIC);
    }
}

Jury jury = SimpleJury.builder()
    .judge(new FileExistsJudge("src/main/java/com/example/Person.java"), 1.0)
    .votingStrategy(new MajorityVotingStrategy())
    .build();

DatasetManager datasetManager = new FileSystemDatasetManager();
ResultStore resultStore = new FileSystemResultStore(Path.of("results"));

ExperimentConfig config = ExperimentConfig.builder()
    .experimentName("rename-field-v1")
    .datasetDir(Path.of("my-dataset"))
    .model("sonnet")
    .promptTemplate("{{task}}")
    .perItemTimeout(Duration.ofMinutes(2))
    .outputDir(Path.of("results"))
    .build();

AgentExperiment experiment = new AgentExperiment(
    datasetManager, jury, resultStore, config);

ExperimentResult result = experiment.run(new MyAgent());

System.out.printf("Pass rate: %.0f%% (%d/%d)%n",
    result.passRate() * 100,
    result.passCount(),
    result.items().size());

// Variant A: base agent
ExperimentConfig configA = ExperimentConfig.builder()
    .experimentName("rename-v1-base")
    .datasetDir(datasetDir)
    .model("sonnet")
    .promptTemplate("{{task}}")
    .perItemTimeout(Duration.ofMinutes(2))
    .build();
ExperimentResult resultA = runner.run(baseAgent);

// Variant B: agent with knowledge base
ExperimentConfig configB = ExperimentConfig.builder()
    .experimentName("rename-v1-with-kb")
    .datasetDir(datasetDir)
    .model("sonnet")
    .promptTemplate("{{task}}\n\nRelevant knowledge:\n{{knowledgeRefs}}")
    .knowledgeBaseDir(Path.of("knowledge"))
    .perItemTimeout(Duration.ofMinutes(2))
    .build();
ExperimentResult resultB = runner.run(kbAgent);