Judge API: Automated Agent Evaluation

public interface Judge {
    Judgment judge(JudgmentContext context);

    default CompletableFuture<Judgment> judgeAsync(JudgmentContext context) {
        return CompletableFuture.supplyAsync(() -> judge(context));
    }
}

public record JudgmentContext(
    String goal,                          // What the agent was asked to do
    Path workspace,                       // Where the agent worked
    Optional<AgentOutput> agentOutput,    // What the agent produced
    Optional<AgentOutput> expectedOutput, // Expected result (if known)
    Optional<List<String>> referenceData, // Context documents
    Instant startedAt,                    // Execution start time
    Duration executionTime,               // How long execution took
    Map<String, Object> metadata          // Additional context
) {}

public record Judgment(
    JudgmentStatus status,  // PASS, FAIL, ABSTAIN, ERROR
    Score score,            // Detailed score (boolean, numerical, categorical)
    String reasoning,       // Explanation of judgment
    List<Check> checks,     // Individual verification checks
    Optional<Duration> elapsed,  // Judgment duration
    Map<String, Object> metadata // Additional metadata
) {
    public boolean pass() {
        return status == JudgmentStatus.PASS;
    }
}

// Quick checks
if (judgment.pass()) { /* ... */ }
if (judgment.fail()) { /* ... */ }
if (judgment.abstain()) { /* ... */ }

// Score access
if (judgment.score() instanceof NumericalScore numerical) {
    double value = numerical.normalized(); // 0.0 to 1.0
}

public sealed interface Score permits BooleanScore, NumericalScore, CategoricalScore {
    Object value();
    ScoreType type();
}

// Boolean: pass/fail
BooleanScore pass = new BooleanScore(true);

// Numerical: scored metrics
NumericalScore quality = new NumericalScore(8.5, 0, 10);
double normalized = quality.normalized(); // 0.85

// Categorical: classification
CategoricalScore level = new CategoricalScore(
    "excellent",
    List.of("poor", "good", "excellent")
);

AgentJudge codeReviewer = AgentJudge.builder()
    .agentClient(agentClient)
    .goal("Review the code for bugs, security issues, and code quality")
    .build();

Judgment review = codeReviewer.judge(context);

@Service
public class ContinuousIntegration {

    private final AgentClient.Builder agentClientBuilder;

    public boolean fixAndDeploy(Path projectRoot) {
        AgentClientResponse response = agentClientBuilder
            .goal("Fix failing tests and run 'mvn clean install'")
            .workingDirectory(projectRoot)
            .advisors(JudgeAdvisor.builder()
                .judge(new BuildSuccessJudge())
                .build())
            .call();

        Judgment judgment = response.getJudgment();

        if (judgment.pass()) {
            deploy(projectRoot);
            return true;
        } else {
            alertTeam("Build failed: " + judgment.reasoning());
            return false;
        }
    }
}

Jury qualityGate = Juries.builder()
    .addJudge("build", new BuildSuccessJudge())
    .addJudge("coverage", new CoverageJudge(80.0))
    .addJudge("correctness", new CorrectnessJudge(chatClient))
    .votingStrategy(VotingStrategies.allMustPass())
    .build();

Verdict verdict = qualityGate.vote(context);

if (!verdict.aggregated().pass()) {
    throw new QualityGateException("Quality standards not met");
}

int maxRetries = 3;
Judgment judgment = null;

for (int attempt = 0; attempt < maxRetries; attempt++) {
    AgentClientResponse response = agentClientBuilder
        .goal("Fix the failing tests")
        .advisors(JudgeAdvisor.builder()
            .judge(new BuildSuccessJudge())
            .build())
        .call();

    judgment = response.getJudgment();

    if (judgment.pass()) {
        break; // Success!
    }

    logger.warn("Attempt {} failed: {}", attempt + 1, judgment.reasoning());
}

if (!judgment.pass()) {
    escalateToHuman(judgment);
}