ui-ux-pro-max-skill/cli/assets/scripts/tests/fixtures/relevance-thresholds.json
Neal Meyer 477bcb28c9
fix: stop Cursor skill pointing at empty Quick Reference (#509)
* fix: stop Cursor skill pointing at empty Quick Reference

Platforms that set quickReference to false omitted the inlined
reference but still told the agent to use "the Quick Reference
sections above." Resolve QR/NOQR variants so Cursor (and other
non-inlined platforms) follow the existing search flow instead.
Claude and Codex keep the inlined reference wording.

Co-authored-by: Neal Meyer <ngmeyer@users.noreply.github.com>

* fix: accept 21 platform configs after ZCode

ZCode landed on main without updating the agent-guide validator
or the platform-count e2e assertion. This branch did not add a
platform; the stale expected count of 20 was failing CI.

Co-authored-by: Neal Meyer <ngmeyer@users.noreply.github.com>

* fix: refresh stale relevance runtime fingerprint

verify:data would fail next on evaluate:relevance because main's
threshold manifest still hashed the pre-#500 runtime. Metrics are
unchanged; only the fingerprint now matches current scripts/data.

Co-authored-by: Neal Meyer <ngmeyer@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Neal Meyer <ngmeyer@users.noreply.github.com>
2026-10-03 23:05:26 +07:00

101 lines
5.0 KiB
JSON

{
"schemaVersion": 1,
"status": "provisional-baseline-regression-gate",
"baselineRevision": "97eb2a2",
"runtimeFingerprint": "48b4cb2aeafb795c76f34bf7faa65e07287811e229b86c152bac3d2cab751385",
"oracleFingerprint": "02e7226428c64a995ec24ae104ce4d186f5f91bf0f062556e90ef9b0fcb6d67a",
"approvingMaintainer": "repository maintainer approved Phase 1 testing checkpoint; second judgment review remains required before final-target promotion",
"units": "All metrics are ratios in [0,1]. Precision treats missing ranks as non-relevant. Negative abstention is measured across the hard-negative by domain/stack cross-product.",
"splitPolicy": {
"calibration": "May be inspected while tuning retrieval.",
"held_out": "Must not be inspected to tune ranking weights; used for final confirmation.",
"varianceAndConfidence": "Corpus is deterministic and has no sampling variance. Report exact ratios and sample counts; do not claim population confidence intervals from this curated set."
},
"tolerancePolicy": "A 1e-12 numeric tolerance permits floating-point representation only. Current known failures remain visible in relevance-baseline.json; the gate prevents regressions and does not claim final quality targets are met.",
"metrics": {
"routingAccuracy": {"floor": 0.875, "tolerance": 1e-12},
"precisionAt1": {"floor": 0.7631578947368421, "tolerance": 1e-12},
"precisionAt3": {"floor": 0.3815789473684211, "tolerance": 1e-12},
"mrrAt3": {"floor": 0.8355263157894737, "tolerance": 1e-12},
"ndcgAt3": {"floor": 0.8366350964266653, "tolerance": 1e-12},
"negativeAbstention": {"floor": 0.9117647058823529, "tolerance": 1e-12},
"typoRecoveryAt3": {"floor": 1.0, "tolerance": 1e-12},
"designSystemCoherence": {"floor": 0.71875, "tolerance": 1e-12}
},
"sampleMinimums": {
"cases": 90,
"retrieval": 76,
"routing": 8,
"negativeChecks": 204,
"typo": 5,
"designSystem": 8,
"domain:style": 2,
"domain:color": 2,
"domain:chart": 2,
"domain:landing": 2,
"domain:product": 2,
"domain:ux": 2,
"domain:typography": 2,
"domain:icons": 2,
"domain:gsap": 2,
"domain:react": 2,
"domain:web": 2,
"domain:google-fonts": 2
},
"splits": {
"calibration": {
"metrics": {
"routingAccuracy": {"floor": 0.8, "tolerance": 1e-12},
"precisionAt1": {"floor": 0.8974358974358975, "tolerance": 1e-12},
"precisionAt3": {"floor": 0.40170940170940167, "tolerance": 1e-12},
"mrrAt3": {"floor": 0.9230769230769231, "tolerance": 1e-12},
"ndcgAt3": {"floor": 0.9103072109232644, "tolerance": 1e-12},
"negativeAbstention": {"floor": 0.9411764705882353, "tolerance": 1e-12},
"typoRecoveryAt3": {"floor": 0.0, "tolerance": 1e-12},
"designSystemCoherence": {"floor": 0.6875, "tolerance": 1e-12}
},
"sampleMinimums": {"cases": 46, "retrieval": 39, "routing": 5, "negativeChecks": 102, "typo": 0, "designSystem": 4}
},
"held_out": {
"metrics": {
"routingAccuracy": {"floor": 1.0, "tolerance": 1e-12},
"precisionAt1": {"floor": 0.6216216216216216, "tolerance": 1e-12},
"precisionAt3": {"floor": 0.36036036036036034, "tolerance": 1e-12},
"mrrAt3": {"floor": 0.7432432432432432, "tolerance": 1e-12},
"ndcgAt3": {"floor": 0.7589807054707906, "tolerance": 1e-12},
"negativeAbstention": {"floor": 0.8823529411764706, "tolerance": 1e-12},
"typoRecoveryAt3": {"floor": 1.0, "tolerance": 1e-12},
"designSystemCoherence": {"floor": 0.75, "tolerance": 1e-12}
},
"sampleMinimums": {"cases": 44, "retrieval": 37, "routing": 3, "negativeChecks": 102, "typo": 5, "designSystem": 4}
}
},
"lockedCases": {
"domain-style-glassmorphism": {"withinTop": 1, "minimumGrade": 2},
"domain-color-spa": {"withinTop": 1, "minimumGrade": 2},
"domain-chart-time-series": {"withinTop": 1, "minimumGrade": 2},
"domain-landing-pricing": {"withinTop": 1, "minimumGrade": 2},
"domain-product-spa": {"withinTop": 1, "minimumGrade": 2},
"domain-ux-keyboard-focus": {"withinTop": 1, "minimumGrade": 2},
"domain-typography-luxury": {"withinTop": 1, "minimumGrade": 2},
"domain-icons-search": {"withinTop": 1, "minimumGrade": 2},
"domain-gsap-scroll-reveal": {"withinTop": 1, "minimumGrade": 2},
"domain-react-parallel-promises": {"withinTop": 1, "minimumGrade": 2},
"domain-web-icon-label": {"withinTop": 1, "minimumGrade": 2},
"domain-google-fonts-inter": {"withinTop": 1, "minimumGrade": 2},
"stack-swiftui-navigation": {"withinTop": 1, "minimumGrade": 2},
"stack-threejs-orbitcontrols": {"withinTop": 1, "minimumGrade": 2},
"stack-uwp-migration": {"withinTop": 1, "minimumGrade": 2},
"stack-winui-dispatcherqueue": {"withinTop": 1, "minimumGrade": 2}
},
"proposedFinalTargets": {
"routingAccuracy": 0.93,
"precisionAt1": 0.8,
"mrrAt3": 0.88,
"ndcgAt3": 0.92,
"negativeAbstention": 0.95,
"typoRecoveryAt3": 0.85,
"designSystemCoherence": 0.9
}
}