fix(eval): distinguish booleans from numbers in snapshot grading (#7358)

Co-authored-by: Cordis798 <284918518+Cordis798@users.noreply.github.com>

backend/scripts/benchmark/context_snapshot/README.md

@@ -61,8 +61,12 @@ with standard computational builtins and a three-second subprocess timeout;
 that grader is not an operating-system sandbox. Run model-generated code in a
 disposable development environment.
 
-`artifact_correct` means all predeclared task checks pass. `clean_success`
-additionally requires a fresh public check of the final revision, completed
+`artifact_correct` means all predeclared task checks pass.
+Grading distinguishes booleans from numbers in scalar and nested JSON/Python
+values, including input-mutation checks; `false` cannot stand in for `0`.
+Ordinary numeric equality, such as `1.0 == 1`, remains unchanged.
+
+`clean_success` additionally requires a fresh public check of the final revision, completed
 executor status, no stop reason and no exception. A correct artifact at the
 turn cap counts only in the first metric. Costs include parent and worker
 requests, including failed/capped runs. Missing provider usage is marked

backend/scripts/benchmark/context_snapshot/grading.py

@@ -18,6 +18,17 @@ if TYPE_CHECKING:
     from .cases import Case
 
 
+def _same_value(actual, expected):
+    """Keep Python value equality, but distinguish booleans from numbers."""
+    if isinstance(actual, bool) != isinstance(expected, bool):
+        return False
+    if isinstance(actual, dict) and isinstance(expected, dict):
+        return actual.keys() == expected.keys() and all(_same_value(actual[key], expected[key]) for key in actual)
+    if isinstance(actual, list) and isinstance(expected, list):
+        return len(actual) == len(expected) and all(_same_value(left, right) for left, right in zip(actual, expected))
+    return actual == expected
+
+
 def grade_python(payload):
     """Run only pure Python over synthetic values, inside a timeout subprocess."""
     code = payload["code"]
@@ -81,10 +92,11 @@ def grade_python(payload):
             original = copy.deepcopy(args)
             try:
                 actual = fn(*args)
-                ok = "raises" not in check and actual == check.get("want") and args == original
-                results.append({"check": number, "passed": ok, "actual": actual, "mutated_input": args != original})
+                unchanged = _same_value(args, original)
+                ok = "raises" not in check and _same_value(actual, check.get("want")) and unchanged
+                results.append({"check": number, "passed": ok, "actual": actual, "mutated_input": not unchanged})
             except Exception as exc:
-                results.append({"check": number, "passed": type(exc).__name__ == check.get("raises") and args == original, "exception": type(exc).__name__})
+                results.append({"check": number, "passed": type(exc).__name__ == check.get("raises") and _same_value(args, original), "exception": type(exc).__name__})
         return {"valid": True, "checks": results}
     except Exception as exc:
         return {"valid": False, "error": f"{type(exc).__name__}: {exc}", "checks": []}
@@ -101,7 +113,7 @@ def score(case: Case, content: str | None, *, public=False, timeout_seconds=3):
             valid = set(value) == set(case.expected)
             if public:
                 return {"valid": valid, "checks": [{"check": "JSON shape", "passed": valid}]}
-            checks = [{"check": name, "passed": value.get(name) == target and isinstance(value.get(name), bool) == isinstance(target, bool)} for name, target in case.expected.items()]
+            checks = [{"check": name, "passed": _same_value(value.get(name), target)} for name, target in case.expected.items()]
             return {"valid": valid, "checks": checks}
         except Exception as exc:
             return {"valid": False, "checks": [], "error": type(exc).__name__}

backend/tests/test_bench_context_snapshot.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 import pytest
 
-from scripts.benchmark.context_snapshot.cases import CASES
+from scripts.benchmark.context_snapshot.cases import CASES, Case
 from scripts.benchmark.context_snapshot.grading import passed, score
 from scripts.benchmark.context_snapshot.report import clean_success, summarize, usage
 
@@ -29,6 +29,37 @@ def test_invoice_grader_allows_divmod_and_detects_rounding_errors():
 """
     assert passed(score(case, correct))
     assert not passed(score(case, correct.replace("r >= 5000", "r > 5000")))
+    assert not passed(score(case, correct.replace("return total", "return total if total else False")))
+
+
+@pytest.mark.parametrize("kind", ["json", "python"])
+@pytest.mark.parametrize(
+    ("actual", "expected"),
+    [(False, 0), (True, 1), ([False], [0]), ({"total": True}, {"total": 1}), ({"rows": [{"done": 0}]}, {"rows": [{"done": False}]})],
+)
+def test_grader_rejects_boolean_numeric_substitutions(kind, actual, expected):
+    case = Case("typed_result", "grading", kind, "", [], expected={"value": expected}, function="answer", checks=[{"args": [], "want": expected}])
+    artifact = json.dumps({"value": actual}) if kind == "json" else f"def answer(): return {actual!r}"
+    assert not passed(score(case, artifact))
+
+
+@pytest.mark.parametrize("kind", ["json", "python"])
+@pytest.mark.parametrize(("actual", "expected"), [(False, False), (True, True), (0, 0), (1.0, 1), ({"values": [True, 1.0]}, {"values": [True, 1]})])
+def test_grader_preserves_valid_booleans_and_numeric_equality(kind, actual, expected):
+    case = Case("typed_result", "grading", kind, "", [], expected={"value": expected}, function="answer", checks=[{"args": [], "want": expected}])
+    artifact = json.dumps({"value": actual}) if kind == "json" else f"def answer(): return {actual!r}"
+    assert passed(score(case, artifact))
+
+
+@pytest.mark.parametrize("raises", [False, True])
+def test_grader_detects_boolean_numeric_input_mutation(raises):
+    check = {"args": [[0]], "raises": "ValueError"} if raises else {"args": [[0]], "want": 7}
+    case = Case("input_mutation", "grading", "python", "", [], function="answer", checks=[check])
+    ending = "raise ValueError()" if raises else "return 7"
+    grade = score(case, f"def answer(values):\n    values[0] = False\n    {ending}")
+    assert not passed(grade)
+    if not raises:
+        assert grade["checks"][0]["mutated_input"]
 
 
 @pytest.mark.parametrize("content", ["[]", "null", '"text"', '{"unexpected":1}', "invalid"])