mirror of
https://github.com/nextlevelbuilder/ui-ux-pro-max-skill.git
synced 2026-08-15 09:18:35 +00:00
a38d04c3d5
Overhaul BM25 relevance, reasoning and data-quality contracts; refresh UI styles and framework guidance; add resilient text, chip, badge and micro-interaction guidance; strengthen release, provenance and catalog refresh gates; update bilingual documentation.
183 lines
8.2 KiB
Python
183 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Unit tests for metric math and relevance fixture validation."""
|
|
|
|
import importlib.util
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = next(parent for parent in Path(__file__).resolve().parents
|
|
if (parent / "scripts/evaluate-relevance.py").exists())
|
|
MODULE_PATH = ROOT / "scripts/evaluate-relevance.py"
|
|
SPEC = importlib.util.spec_from_file_location("evaluate_relevance", MODULE_PATH)
|
|
evaluator = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(evaluator)
|
|
|
|
|
|
class TestMetricMath(unittest.TestCase):
|
|
def test_precision_counts_missing_ranks_as_non_relevant(self):
|
|
self.assertEqual(evaluator.precision_at_k([2], 1), 1.0)
|
|
self.assertAlmostEqual(evaluator.precision_at_k([2], 3), 1 / 3)
|
|
self.assertEqual(evaluator.precision_at_k([], 3), 0.0)
|
|
self.assertEqual(evaluator.precision_at_k([2], 0), 0.0)
|
|
|
|
def test_reciprocal_rank_stops_at_k(self):
|
|
self.assertEqual(evaluator.reciprocal_rank([0, 2, 0]), 0.5)
|
|
self.assertEqual(evaluator.reciprocal_rank([0, 0, 0, 2]), 0.0)
|
|
self.assertEqual(evaluator.reciprocal_rank([]), 0.0)
|
|
|
|
def test_ndcg_uses_graded_gain_and_handles_empty_ideal(self):
|
|
self.assertEqual(evaluator.ndcg_at_k([2, 1], [2, 1]), 1.0)
|
|
self.assertLess(evaluator.ndcg_at_k([1, 2], [2, 1]), 1.0)
|
|
self.assertEqual(evaluator.ndcg_at_k([], [], 3), 0.0)
|
|
|
|
def test_result_grades_match_identity_subsets(self):
|
|
results = [
|
|
{"Category": "State", "Guideline": "Use useState", "Severity": "Medium"},
|
|
{"Category": "State", "Guideline": "Use useReducer", "Severity": "Medium"},
|
|
]
|
|
judgments = [
|
|
{"identity": {"Guideline": "Use useReducer"}, "grade": 2},
|
|
{"identity": {"Category": "State"}, "grade": 1},
|
|
]
|
|
self.assertEqual(evaluator.grades_for_results(results, judgments), [1, 2])
|
|
|
|
|
|
class TestFixtureValidation(unittest.TestCase):
|
|
@staticmethod
|
|
def valid_fixture():
|
|
case = {
|
|
"id": "domain-style-minimal",
|
|
"split": "calibration",
|
|
"mode": "domain",
|
|
"domain": "style",
|
|
"query": "minimal grid",
|
|
"judgments": [{"identity": {"Style Category": "Minimalism"}, "grade": 2}],
|
|
}
|
|
return {
|
|
"schemaVersion": 1,
|
|
"globalNegativeApplicability": {"domains": ["style"], "stacks": []},
|
|
"cases": [dict(case, id=f"case-{index}") for index in range(60)],
|
|
}
|
|
|
|
def test_valid_schema(self):
|
|
self.assertEqual(evaluator.validate_fixture(self.valid_fixture(), {"style": {}}, []), [])
|
|
|
|
def test_rejects_bad_count_duplicate_id_and_grade(self):
|
|
fixture = self.valid_fixture()
|
|
fixture["cases"] = fixture["cases"][:2]
|
|
fixture["cases"][1]["id"] = fixture["cases"][0]["id"]
|
|
fixture["cases"][0]["judgments"][0]["grade"] = 3
|
|
errors = "\n".join(evaluator.validate_fixture(fixture, {"style": {}}, []))
|
|
self.assertIn("60-100", errors)
|
|
self.assertIn("duplicate case id", errors)
|
|
self.assertIn("grade 1 or 2", errors)
|
|
|
|
|
|
class TestThresholdGate(unittest.TestCase):
|
|
def test_runtime_fingerprint_binds_reasoning_contract(self):
|
|
original = evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
runtime = root / "src/ui-ux-pro-max/scripts"
|
|
data = root / "src/ui-ux-pro-max/data"
|
|
runtime.mkdir(parents=True)
|
|
data.mkdir(parents=True)
|
|
for name in ("core.py", "design_system.py", "reasoning_contract.py"):
|
|
(runtime / name).write_text(name, encoding="utf-8")
|
|
(data / "styles.csv").write_text("No,Style\n1,Test\n", encoding="utf-8")
|
|
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = root, runtime, data
|
|
try:
|
|
before = evaluator.runtime_fingerprint()
|
|
(runtime / "reasoning_contract.py").write_text("changed", encoding="utf-8")
|
|
self.assertNotEqual(before, evaluator.runtime_fingerprint())
|
|
finally:
|
|
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = original
|
|
|
|
def test_oracle_fingerprint_hashes_the_selected_cases_file(self):
|
|
canonical = evaluator.FIXTURE_DIR / "relevance-cases.json"
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
selected = Path(tmp) / "cases.json"
|
|
selected.write_bytes(canonical.read_bytes())
|
|
self.assertEqual(
|
|
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
|
selected.write_bytes(canonical.read_bytes() + b" ")
|
|
self.assertNotEqual(
|
|
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
|
|
|
def test_metric_sample_and_locked_case_failures_are_actionable(self):
|
|
report = {
|
|
"metrics": {"precisionAt1": 0.5},
|
|
"samples": {"retrieval": 1},
|
|
"cases": [{"id": "locked", "grades": [0], "actual": [{"Style Category": "Wrong"}]}],
|
|
}
|
|
manifest = {
|
|
"metrics": {"precisionAt1": {"floor": 0.8, "tolerance": 0.01}},
|
|
"sampleMinimums": {"retrieval": 2},
|
|
"lockedCases": {"locked": {"withinTop": 1, "minimumGrade": 2}},
|
|
}
|
|
manifest["splits"] = {"calibration": {"metrics": {}, "sampleMinimums": {}},
|
|
"held_out": {"metrics": {}, "sampleMinimums": {}}}
|
|
report["splits"] = {"calibration": {"metrics": {}, "samples": {}},
|
|
"held_out": {"metrics": {}, "samples": {}}}
|
|
failures = evaluator.check_thresholds(report, manifest)
|
|
self.assertEqual(len(failures), 3)
|
|
self.assertTrue(any("Wrong" in failure for failure in failures))
|
|
|
|
def test_manifest_rejects_missing_contract_sections(self):
|
|
errors = evaluator.validate_manifest({}, "fingerprint")
|
|
self.assertTrue(any("missing sections" in error for error in errors))
|
|
self.assertTrue(any("missing metrics" in error for error in errors))
|
|
|
|
def test_manifest_rejects_non_finite_and_invalid_sample_values(self):
|
|
manifest = {
|
|
"schemaVersion": 1,
|
|
"status": "approved",
|
|
"approvingMaintainer": "maintainer",
|
|
"units": "ratios",
|
|
"splitPolicy": {},
|
|
"runtimeFingerprint": "fingerprint",
|
|
"oracleFingerprint": "oracle",
|
|
"baselineRevision": "97eb2a2",
|
|
"metrics": {name: {"floor": float("nan")} for name in evaluator.REQUIRED_METRICS},
|
|
"sampleMinimums": {"cases": True},
|
|
"lockedCases": {"case": {}},
|
|
"splits": {
|
|
split: {
|
|
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
|
"sampleMinimums": {"cases": 1},
|
|
} for split in ("calibration", "held_out")
|
|
},
|
|
}
|
|
errors = evaluator.validate_manifest(manifest, "fingerprint", "oracle")
|
|
self.assertTrue(any("finite" in error for error in errors))
|
|
self.assertTrue(any("non-negative integer" in error for error in errors))
|
|
|
|
def test_manifest_binds_oracle_and_validates_baseline_revision(self):
|
|
manifest = {
|
|
"schemaVersion": 1,
|
|
"status": "approved",
|
|
"approvingMaintainer": "maintainer",
|
|
"units": "ratios",
|
|
"splitPolicy": {},
|
|
"runtimeFingerprint": "runtime",
|
|
"oracleFingerprint": "wrong",
|
|
"baselineRevision": "not-a-revision",
|
|
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
|
"sampleMinimums": {"cases": 1},
|
|
"lockedCases": {"case": {}},
|
|
"splits": {
|
|
split: {
|
|
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
|
"sampleMinimums": {"cases": 1},
|
|
} for split in ("calibration", "held_out")
|
|
},
|
|
}
|
|
errors = evaluator.validate_manifest(manifest, "runtime", "expected")
|
|
self.assertTrue(any("oracleFingerprint" in error for error in errors))
|
|
self.assertTrue(any("baselineRevision" in error for error in errors))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|