API Reference

@FunctionalInterface
public interface Judge {
    Judgment judge(JudgmentContext context);
}

public interface AsyncJudge {
    CompletableFuture<Judgment> judgeAsync(JudgmentContext context);
}

public interface ReactiveJudge {
    Mono<Judgment> judge(JudgmentContext context);
}

public abstract class DeterministicJudge implements JudgeWithMetadata {
    protected DeterministicJudge(String name, String description)
    public JudgeMetadata metadata()
}

Judge named = Judges.named(myLambda, "check-name", "description");
Judge typed = Judges.named(myLambda, "check-name", "description", JudgeType.DETERMINISTIC);

public interface JudgeWithMetadata extends Judge {
    JudgeMetadata metadata();
}

if (judge instanceof JudgeWithMetadata jwm) {
    log.info("Running: {}", jwm.metadata().name());
}

public record JudgeMetadata(String name, String description, JudgeType type)

public enum JudgeType {
    DETERMINISTIC,
    LLM_POWERED,
    HYBRID,
    AGENT
}

public record JudgmentContext(
    String goal,
    Path workspace,
    Duration executionTime,
    Instant startedAt,
    Optional<String> agentOutput,
    ExecutionStatus status,
    Optional<Throwable> error,
    Map<String, Object> metadata
)

JudgmentContext context = JudgmentContext.builder()
    .goal("Add REST endpoint")
    .workspace(Path.of("/project"))
    .status(ExecutionStatus.SUCCESS)
    .startedAt(Instant.now())
    .executionTime(Duration.ofMinutes(2))
    .agentOutput("Created HelloController.java")
    .metadata("expectedDir", Path.of("/reference"))
    .build();

public enum ExecutionStatus {
    SUCCESS, FAILED, TIMEOUT, CANCELLED, REFUSED, UNKNOWN
}

public record Judgment(
    Score score,
    JudgmentStatus status,
    String reasoning,
    List<Check> checks,
    Map<String, Object> metadata
)

Judgment.pass("Build succeeded")
Judgment.fail("File not found")
Judgment.abstain("Not enough information")
Judgment.error("Timeout", exception)

Judgment.builder()
    .score(new BooleanScore(true))
    .status(JudgmentStatus.PASS)
    .reasoning("All checks passed")
    .check(Check.pass("compile", "Compilation succeeded"))
    .check(Check.pass("tests", "All tests passed"))
    .metadata("duration", Duration.ofSeconds(45))
    .build();

public enum JudgmentStatus {
    PASS, FAIL, ABSTAIN, ERROR
}

public record Check(String name, boolean passed, String message)

Check.pass("file_exists")
Check.pass("file_exists", "File found at expected path")
Check.fail("content_match", "Expected header not found")

public record BooleanScore(boolean value) implements Score

new BooleanScore(true)   // pass
new BooleanScore(false)  // fail

public record NumericalScore(double value, double min, double max) implements Score

NumericalScore score = new NumericalScore(85.0, 0.0, 100.0);
double normalized = score.normalized();  // 0.85 (scaled to [0, 1])

// Convenience factories
NumericalScore zeroToOne = NumericalScore.normalized(0.85);
NumericalScore zeroToTen = NumericalScore.outOfTen(7.5);

public record CategoricalScore(String value, List<String> allowedValues) implements Score

CategoricalScore score = new CategoricalScore(
    "GOOD", List.of("EXCELLENT", "GOOD", "FAIR", "POOR"));

// Numerical and Boolean scores normalize directly
// Categorical scores require a mapping from category names to numeric values
double normalized = Scores.toNormalized(anyScore, categoryMap);

ModelBackedJudge judge = ModelBackedJudge.builder()
    .model(judgeModel)
    .template(promptTemplate)
    .classifier(LabelJudgmentClassifier.passFail())
    .build();

Judgment judgment = judge.judge(context);

@FunctionalInterface
public interface JudgeModel {
    JudgeModelResponse call(JudgeModelRequest request);
}

JudgePromptTemplate template = JudgePromptTemplate.builder()
    .source(TextSource.classpath("/prompts/correctness.txt"))
    .renderer(new SimpleJudgeTemplateRenderer())
    .missingVariablePolicy(MissingVariablePolicy.STRICT)
    .build();

public interface JudgeTemplateRenderer {
    String render(String template, Map<String, String> variables);
}

@FunctionalInterface
public interface JudgmentClassifier {
    Judgment classify(JudgeModelResponse response);
}

// Built-in: maps "PASS"/"FAIL" labels to Judgment
JudgmentClassifier classifier = LabelJudgmentClassifier.passFail();

// Custom labels via builder
JudgmentClassifier classifier = LabelJudgmentClassifier.builder()
    .passLabel("CORRECT")
    .failLabel("INCORRECT")
    .build();

// Model request — messages with options and metadata
public record JudgeModelRequest(
    List<JudgeMessage> messages,
    Map<String, Object> options,
    Map<String, Object> metadata
)
// Factory: JudgeModelRequest.user(prompt)

// Model response — text, model identity, usage, metadata
public record JudgeModelResponse(
    String text,
    String model,
    Usage usage,
    Map<String, Object> metadata
)

// Message with role
public record JudgeMessage(JudgeMessageRole role, String content)
public enum JudgeMessageRole { SYSTEM, USER, ASSISTANT }

// Token usage
public record Usage(int inputTokens, int outputTokens, int totalTokens,
                    double estimatedCostUsd)

public interface Jury {
    Verdict vote(JudgmentContext context);
}

public record Verdict(
    Judgment aggregated,
    List<Judgment> individual,
    Map<String, Judgment> individualByName,
    Map<String, Double> weights,
    List<Verdict> subVerdicts
)

public interface VotingStrategy {
    Judgment aggregate(List<Judgment> judgments, Map<String, Double> weights);
    String getName();
}

public enum TierPolicy {
    REJECT_ON_ANY_FAIL,   // Stop on failure
    ACCEPT_ON_ALL_PASS,   // Stop on full pass
    FINAL_TIER            // Always produces verdict (required for last tier)
}

public enum TiePolicy {
    PASS,     // Optimistic
    FAIL,     // Pessimistic (default)
    ABSTAIN   // Neutral
}

public enum ErrorPolicy {
    TREAT_AS_FAIL,     // Default
    TREAT_AS_ABSTAIN,
    IGNORE
}

// Quick jury from judges
Jury jury = Juries.fromJudges(strategy, judge1, judge2, judge3);

// Combine two juries into a meta-jury
Jury meta = Juries.combine(jury1, jury2, strategy);

// Multiple juries
Jury all = Juries.allOf(strategy, jury1, jury2, jury3);

// One-liner with a judge
Judgment result = SpringAiEvaluator.evaluate(
    "Summarize the document",
    () -> chatClient.prompt().user(prompt).call().chatResponse(),
    myJudge);

// One-liner with a jury
Verdict verdict = SpringAiEvaluator.evaluate(
    "Summarize the document",
    () -> chatClient.prompt().user(prompt).call().chatResponse(),
    myJury);

Judgment result = LangChain4jEvaluator.evaluate(
    "Summarize the document",
    goal -> assistant.chat(goal),
    myJudge);

Judgment result = KoogEvaluator.evaluate(agent, "Summarize the document", myJudge);

Judgment result = AgentClientEvaluator.evaluate(
    "Fix the build", workspace,
    () -> agentClient.run("Fix the build"),
    myJudge);

// Build context from a pre-existing response
JudgmentContext ctx = SpringAiJudgmentContextBuilder.from(
    chatResponse, "goal", startedAt, duration);

// Or execute and capture in one step
JudgmentContext ctx = SpringAiJudgmentContextBuilder.execute(
    "goal", () -> chatClient.prompt().user(prompt).call().chatResponse());

String question = RagContext.question(context);       // rag.question or goal
Optional<String> ctx = RagContext.context(context);   // rag.context or langchain4j.sources
Optional<String> answer = RagContext.answer(context); // rag.answer or agentOutput

<dependency>
    <groupId>io.github.markpollack</groupId>
    <artifactId>agent-judge-core</artifactId>
    <version>0.11.0</version>
</dependency>