diff --git a/backend/scripts/benchmark/context_snapshot/README.md b/backend/scripts/benchmark/context_snapshot/README.md index 944e2f2e87..89cd7b4dd1 100644 --- a/backend/scripts/benchmark/context_snapshot/README.md +++ b/backend/scripts/benchmark/context_snapshot/README.md @@ -61,8 +61,12 @@ with standard computational builtins and a three-second subprocess timeout; that grader is not an operating-system sandbox. Run model-generated code in a disposable development environment. -`artifact_correct` means all predeclared task checks pass. `clean_success` -additionally requires a fresh public check of the final revision, completed +`artifact_correct` means all predeclared task checks pass. +Grading distinguishes booleans from numbers in scalar and nested JSON/Python +values, including input-mutation checks; `false` cannot stand in for `0`. +Ordinary numeric equality, such as `1.0 == 1`, remains unchanged. + +`clean_success` additionally requires a fresh public check of the final revision, completed executor status, no stop reason and no exception. A correct artifact at the turn cap counts only in the first metric. Costs include parent and worker requests, including failed/capped runs. Missing provider usage is marked diff --git a/backend/scripts/benchmark/context_snapshot/grading.py b/backend/scripts/benchmark/context_snapshot/grading.py index 9143190d32..7d2fa3c353 100644 --- a/backend/scripts/benchmark/context_snapshot/grading.py +++ b/backend/scripts/benchmark/context_snapshot/grading.py @@ -18,6 +18,17 @@ if TYPE_CHECKING: from .cases import Case +def _same_value(actual, expected): + """Keep Python value equality, but distinguish booleans from numbers.""" + if isinstance(actual, bool) != isinstance(expected, bool): + return False + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all(_same_value(actual[key], expected[key]) for key in actual) + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all(_same_value(left, right) for left, right in zip(actual, expected)) + return actual == expected + + def grade_python(payload): """Run only pure Python over synthetic values, inside a timeout subprocess.""" code = payload["code"] @@ -81,10 +92,11 @@ def grade_python(payload): original = copy.deepcopy(args) try: actual = fn(*args) - ok = "raises" not in check and actual == check.get("want") and args == original - results.append({"check": number, "passed": ok, "actual": actual, "mutated_input": args != original}) + unchanged = _same_value(args, original) + ok = "raises" not in check and _same_value(actual, check.get("want")) and unchanged + results.append({"check": number, "passed": ok, "actual": actual, "mutated_input": not unchanged}) except Exception as exc: - results.append({"check": number, "passed": type(exc).__name__ == check.get("raises") and args == original, "exception": type(exc).__name__}) + results.append({"check": number, "passed": type(exc).__name__ == check.get("raises") and _same_value(args, original), "exception": type(exc).__name__}) return {"valid": True, "checks": results} except Exception as exc: return {"valid": False, "error": f"{type(exc).__name__}: {exc}", "checks": []} @@ -101,7 +113,7 @@ def score(case: Case, content: str | None, *, public=False, timeout_seconds=3): valid = set(value) == set(case.expected) if public: return {"valid": valid, "checks": [{"check": "JSON shape", "passed": valid}]} - checks = [{"check": name, "passed": value.get(name) == target and isinstance(value.get(name), bool) == isinstance(target, bool)} for name, target in case.expected.items()] + checks = [{"check": name, "passed": _same_value(value.get(name), target)} for name, target in case.expected.items()] return {"valid": valid, "checks": checks} except Exception as exc: return {"valid": False, "checks": [], "error": type(exc).__name__} diff --git a/backend/tests/test_bench_context_snapshot.py b/backend/tests/test_bench_context_snapshot.py index 76473cf6c1..d88ef809cc 100644 --- a/backend/tests/test_bench_context_snapshot.py +++ b/backend/tests/test_bench_context_snapshot.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from scripts.benchmark.context_snapshot.cases import CASES +from scripts.benchmark.context_snapshot.cases import CASES, Case from scripts.benchmark.context_snapshot.grading import passed, score from scripts.benchmark.context_snapshot.report import clean_success, summarize, usage @@ -29,6 +29,37 @@ def test_invoice_grader_allows_divmod_and_detects_rounding_errors(): """ assert passed(score(case, correct)) assert not passed(score(case, correct.replace("r >= 5000", "r > 5000"))) + assert not passed(score(case, correct.replace("return total", "return total if total else False"))) + + +@pytest.mark.parametrize("kind", ["json", "python"]) +@pytest.mark.parametrize( + ("actual", "expected"), + [(False, 0), (True, 1), ([False], [0]), ({"total": True}, {"total": 1}), ({"rows": [{"done": 0}]}, {"rows": [{"done": False}]})], +) +def test_grader_rejects_boolean_numeric_substitutions(kind, actual, expected): + case = Case("typed_result", "grading", kind, "", [], expected={"value": expected}, function="answer", checks=[{"args": [], "want": expected}]) + artifact = json.dumps({"value": actual}) if kind == "json" else f"def answer(): return {actual!r}" + assert not passed(score(case, artifact)) + + +@pytest.mark.parametrize("kind", ["json", "python"]) +@pytest.mark.parametrize(("actual", "expected"), [(False, False), (True, True), (0, 0), (1.0, 1), ({"values": [True, 1.0]}, {"values": [True, 1]})]) +def test_grader_preserves_valid_booleans_and_numeric_equality(kind, actual, expected): + case = Case("typed_result", "grading", kind, "", [], expected={"value": expected}, function="answer", checks=[{"args": [], "want": expected}]) + artifact = json.dumps({"value": actual}) if kind == "json" else f"def answer(): return {actual!r}" + assert passed(score(case, artifact)) + + +@pytest.mark.parametrize("raises", [False, True]) +def test_grader_detects_boolean_numeric_input_mutation(raises): + check = {"args": [[0]], "raises": "ValueError"} if raises else {"args": [[0]], "want": 7} + case = Case("input_mutation", "grading", "python", "", [], function="answer", checks=[check]) + ending = "raise ValueError()" if raises else "return 7" + grade = score(case, f"def answer(values):\n values[0] = False\n {ending}") + assert not passed(grade) + if not raises: + assert grade["checks"][0]["mutated_input"] @pytest.mark.parametrize("content", ["[]", "null", '"text"', '{"unexpected":1}', "invalid"])