Evaluation API¶

def run_openrouter_evaluation(
    examples: list[DecisionExample],
    output_dir: Path,
    *,
    model: str,
    reasoning_effort: str,
    seed: int,
    concurrency: int,
    reasoning_family_effort: str | None = None,
    benchmark_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Evaluate rows concurrently with append-only raw output and resumability."""

    with OpenRouterDecisionModel(
        model=model,
        reasoning_effort=reasoning_effort,
        reasoning_family_effort=reasoning_family_effort,
        seed=seed,
    ) as decision_model:
        return _run_evaluation(
            examples,
            output_dir,
            decision_model=decision_model,
            concurrency=concurrency,
            metadata={
                "model": model,
                "reasoning_effort": reasoning_effort,
                "reasoning_family_effort": reasoning_family_effort,
                "reasoning_effort_policy": (
                    "reasoning-family-override-v1"
                    if reasoning_family_effort is not None
                    else "uniform-v1"
                ),
                "seed": seed,
                "prompt_version": PROMPT_VERSION,
                "prompt_sha256": prompt_sha256(),
                "prediction_normalization": "divide_positive_finite_values_by_sum",
                **(benchmark_metadata or {}),
            },
        )

def run_jev_openrouter_evaluation(
    examples: list[DecisionExample],
    output_dir: Path,
    *,
    model: str,
    concurrency: int,
    max_state_question_tokens: int = 32_000,
    input_token_reserve: int = 2_048,
    tokenizer_model: str = "Qwen/Qwen3-0.6B",
    tokenizer_revision: str = "c1899de289a04d12100db370d81485cdf75e47ca",
    benchmark_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Evaluate Jev through OpenRouter's native Decisions endpoint."""

    with JevOpenRouterDecisionModel(
        model=model,
        max_state_question_tokens=max_state_question_tokens,
        input_token_reserve=input_token_reserve,
        tokenizer_model=tokenizer_model,
        tokenizer_revision=tokenizer_revision,
    ) as decision_model:
        return _run_evaluation(
            examples,
            output_dir,
            decision_model=decision_model,
            concurrency=concurrency,
            metadata={
                "model": model,
                "native_contract_version": JEV_CONTRACT_VERSION,
                "prediction_normalization": "native_probability_distribution",
                **decision_model.metadata,
                **(benchmark_metadata or {}),
            },
        )

def run_hf_evaluation(
    examples: list[DecisionExample],
    output_dir: Path,
    *,
    model_dir: Path,
    seed: int,
    batch_size: int,
    max_prompt_characters_per_batch: int,
    max_length: int,
    attn_implementation: str,
    benchmark_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Evaluate a native local HF decision-token checkpoint in CUDA batches."""

    decision_model = HFDecisionModel(
        model_dir=model_dir,
        seed=seed,
        max_length=max_length,
        attn_implementation=attn_implementation,
    )
    return _run_hf_batches(
        examples,
        output_dir,
        decision_model=decision_model,
        batch_size=batch_size,
        max_prompt_characters_per_batch=max_prompt_characters_per_batch,
        metadata={**decision_model.metadata, **(benchmark_metadata or {})},
    )

def summarize_raw(raw_path: Path) -> dict[str, Any]:
    """Aggregate successful raw rows without hiding failures."""

    groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
    latest_by_row_id: dict[str, dict[str, Any]] = {}
    with raw_path.open() as handle:
        for line in handle:
            record = json.loads(line)
            latest_by_row_id[str(record["row_id"])] = record
    errors = 0
    for record in latest_by_row_id.values():
        if record["status"] != "ok":
            errors += 1
            continue
        groups["overall"].append(record)
        task_name = _record_dimension(record, "task_name")
        if task_name is not None:
            groups[f"task:{task_name}"].append(record)
        groups[f"primitive:{record['primitive']}"].append(record)
        groups[f"family:{record['family']}"].append(record)
        groups[f"domain:{record['domain']}"].append(record)
        candidate_count = int(record.get("candidate_count", len(record["scored"]["probabilities"])))
        groups[f"candidate_count:{candidate_count}"].append(record)
    input_contracts = [
        record["input_contract"]
        for record in groups["overall"]
        if isinstance(record.get("input_contract"), dict)
    ]
    summary: dict[str, Any] = {
        "successful_rows": len(groups["overall"]),
        "error_rows": errors,
        "model_input_truncation": {
            "reported_rows": len(input_contracts),
            "truncated_rows": sum(
                bool(contract.get("truncated")) for contract in input_contracts
            ),
            "policy_versions": sorted(
                {
                    str(contract["policy_version"])
                    for contract in input_contracts
                    if "policy_version" in contract
                }
            ),
        },
        "metrics": {
            name: {
                "rows": len(records),
                "accuracy": sum(bool(record["scored"]["correct"]) for record in records)
                / len(records),
                "mean_negative_log_likelihood": sum(
                    float(record["negative_log_likelihood"]) for record in records
                )
                / len(records),
                "mean_latency_seconds": sum(float(record["latency_seconds"]) for record in records)
                / len(records),
                "expected_calibration_error": expected_calibration_error(records),
                "ece_bins": ECE_BINS,
            }
            for name, records in sorted(groups.items())
            if records
        },
    }

    summary["metrics"].setdefault("overall", {"rows": 0})
    return summary