"""Line-oriented `secret-env-assignment` sweep: every spelling of the same binding. The rule runs over text (`_scan_secret_assignments_by_text`), so the *format* must not decide whether a credential is reported: `api_key: ` in YAML, `API_KEY=` in `.env`/`.ini`/shell and `{"api_key": ""}` in JSON are the same binding and have to read the same way. The quoted-key shapes below are what a JSON config uses. """ from __future__ import annotations import json from pathlib import Path import pytest from deerflow.skills.skillscan.orchestrator import _SECRET_ASSIGNMENT_RE, scan_skill_dir SECRET = "9f8e7d6c5b4a3210ff" # Key on the left, value on the right, with the key quoted the way each format spells it. QUOTED_KEY_CASES = [ ("config.json", '{"api_key": "%s"}'), ("token.json", '{"token": "%s"}'), ("plural.json", '{"secrets": "%s"}'), ("password.json", '{"password": "%s"}'), ("hyphen.json", '{"api-key": "%s"}'), ("single.json", "{'api_key': '%s'}"), ("spaced.json", '{"api_key" : "%s"}'), ("nested.json", '{"auth": {"api_key": "%s"}}'), ("inline.json", '{"api_key": "%s", "other": 1}'), ] UNQUOTED_KEY_CASES = [ ("config.yaml", "api_key: %s"), ("config.ini", "api_key=%s"), ("app.env", "API_KEY=%s"), ] # Shapes the rule must keep quiet about, so the quoted-key match does not become a blanket # "any JSON string" match. QUIET_CASES = [ ("placeholder.json", '{"api_key": "your-key-here"}'), ("env-ref.json", '{"api_key": "${API_KEY}"}'), ("empty.json", '{"api_key": ""}'), ("unrelated.json", '{"other": "%s"}'), ] # The credential word is often the tail of a longer name separated by `_`, `.` or `-` # (`access_token`, `client_secret`, `x-api-key`). Those are the common spellings in OAuth # and service-account config, and each must read the same as the bare name. SUFFIXED_KEY_CASES = [ ("oauth.json", '{"access_token": "%s"}'), ("client.json", '{"client_secret": "%s"}'), ("refresh.json", '{"refresh_token": "%s"}'), ("access.env", "ACCESS_TOKEN=%s"), ("client.env", "CLIENT_SECRET=%s"), ("api.env", "MY_API_KEY=%s"), ("client.yaml", "client_secret: %s"), ("access.ini", "access_token=%s"), ("dotted.yaml", "auth.token: %s"), ("hyphen.yaml", "x-api-key: %s"), ] # A run of letters before the credential word is not a credential name: the sweep must not # fire on ordinary words that merely contain the keyword (`tokenizer`, `secretive`). MIDWORD_QUIET_CASES = [ ("tokenizer.yaml", "tokenizer: %s"), ("secretive.yaml", "secretive: %s"), ("passwordless.yaml", "passwordless: %s"), ("credentials_file.yaml", "credentials_file: %s"), ("mytoken.yaml", "mytoken: %s"), ] def _write_package(parent: Path, filename: str, text: str) -> Path: root = parent / "pkg" root.mkdir(parents=True, exist_ok=True) (root / "SKILL.md").write_text("---\nname: demo\ndescription: A demo skill\n---\n\nBody.\n", encoding="utf-8") (root / filename).write_text(text + "\n", encoding="utf-8") return root def _secret_findings(root: Path) -> list[dict]: return [finding for finding in scan_skill_dir(root)["findings"] if finding["rule_id"] == "secret-env-assignment"] @pytest.mark.parametrize(("filename", "template"), QUOTED_KEY_CASES) def test_quoted_key_is_reported(tmp_path: Path, filename: str, template: str) -> None: root = _write_package(tmp_path, filename, template % SECRET) assert [(finding["rule_id"], finding["line"]) for finding in _secret_findings(root)] == [("secret-env-assignment", 1)] @pytest.mark.parametrize(("filename", "template"), UNQUOTED_KEY_CASES) def test_unquoted_key_stays_reported(tmp_path: Path, filename: str, template: str) -> None: root = _write_package(tmp_path, filename, template % SECRET) assert [(finding["rule_id"], finding["line"]) for finding in _secret_findings(root)] == [("secret-env-assignment", 1)] @pytest.mark.parametrize(("filename", "template"), SUFFIXED_KEY_CASES) def test_separator_prefixed_key_is_reported(tmp_path: Path, filename: str, template: str) -> None: root = _write_package(tmp_path, filename, template % SECRET) assert [(finding["rule_id"], finding["line"]) for finding in _secret_findings(root)] == [("secret-env-assignment", 1)] @pytest.mark.parametrize(("filename", "template"), MIDWORD_QUIET_CASES) def test_midword_key_stays_quiet(tmp_path: Path, filename: str, template: str) -> None: root = _write_package(tmp_path, filename, template % SECRET) assert _secret_findings(root) == [] @pytest.mark.parametrize( "text", [ '{"access_token": "' + SECRET + '"}', '{"client_secret": "' + SECRET + '"}', "ACCESS_TOKEN=" + SECRET, "MY_API_KEY=" + SECRET, ], ) def test_sweep_matches_separator_prefixed_keys(text: str) -> None: """The review's shapes: a separator may introduce the credential word.""" assert [match.group(2) for match in _SECRET_ASSIGNMENT_RE.finditer(text)] == [SECRET] @pytest.mark.parametrize("word", ["tokenizer", "secretive", "passwordless", "mytoken"]) def test_sweep_ignores_a_letter_run_before_the_word(word: str) -> None: """A letter run is not a separator, so the word boundary still holds.""" assert list(_SECRET_ASSIGNMENT_RE.finditer(f"{word}: {SECRET}")) == [] @pytest.mark.parametrize(("filename", "template"), QUIET_CASES) def test_non_credential_or_placeholder_values_stay_quiet(tmp_path: Path, filename: str, template: str) -> None: root = _write_package(tmp_path, filename, template % SECRET if "%s" in template else template) assert _secret_findings(root) == [] def test_json_key_spelling_reports_the_same_finding_as_yaml(tmp_path: Path) -> None: """The format alone must not decide whether the binding is reported.""" yaml_root = _write_package(tmp_path / "yaml", "config.yaml", f"api_key: {SECRET}") json_root = _write_package(tmp_path / "json", "config.json", '{"api_key": "' + SECRET + '"}') assert [finding["line"] for finding in _secret_findings(yaml_root)] == [1] assert [finding["line"] for finding in _secret_findings(json_root)] == [1] def test_pretty_printed_json_reports_the_line_the_key_is_on(tmp_path: Path) -> None: root = _write_package(tmp_path, "config.json", '{\n "api_key": "' + SECRET + '"\n}') assert [finding["line"] for finding in _secret_findings(root)] == [2] @pytest.mark.parametrize("name", ["tokens", "total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens", "cached_tokens", "reasoning_tokens"]) @pytest.mark.parametrize("value", [0, 84852]) @pytest.mark.parametrize("template", ['{"%s": %s}', '{\n "%s": %s,\n "duration_ms": 23332\n}', "%s: %s", "%s=%s"]) def test_numeric_model_token_counts_are_not_credentials(tmp_path: Path, name: str, value: int, template: str) -> None: root = _write_package(tmp_path, "measurements.md", template % (name.upper() if "=" in template else name, value)) assert _secret_findings(root) == [] @pytest.mark.parametrize( "text", [ '{"total_tokens": "84852"}', '{"total_tokens": "' + SECRET + '"}', "total_tokens: 84852secret", "access_tokens: 84852", "api_key: 84852", "password: 84852", "access_total_tokens: 84852", '{"access total_tokens": 84852}', '{"access/total_tokens": 84852}', '{"tokens": "84852"}', '{"tokens": {"mean": "' + SECRET + '", "stddev": 1, "min": 0, "max": 2}}', '{"tokens": {"api_key": "' + SECRET + '"}}', '{"tokens": {"mean": true, "stddev": 1, "min": 0, "max": 2}}', '{"tokens": {"mean": NaN, "stddev": 1, "min": 0, "max": 2}}', '{"tokens": {"mean": 1e999, "stddev": 1, "min": 0, "max": 2}}', '{"tokens": {"mean": "' + SECRET + '", "mean": 1, "stddev": 1, "min": 0, "max": 2}}', '{"tokens": [["mean", 1], ["stddev", 1], ["min", 0], ["max", 2]]}', '{"tokens": "+1700' + SECRET + '"}', ], ) def test_token_count_exemption_preserves_credential_detection(tmp_path: Path, text: str) -> None: root = _write_package(tmp_path, "config.md", text) assert [finding["line"] for finding in _secret_findings(root)] == [1] def test_numeric_token_count_does_not_hide_later_credential(tmp_path: Path) -> None: root = _write_package(tmp_path, "config.md", '{"total_tokens": 84852}\napi_key: ' + SECRET) assert [finding["line"] for finding in _secret_findings(root)] == [2] def test_numeric_token_count_does_not_suppress_cloud_token_scan(tmp_path: Path) -> None: token = "sk-" + "a1b2c3d4" * 4 root = _write_package(tmp_path, "config.md", '{"total_tokens": 84852}\nAuthorization: Bearer ' + token) findings = scan_skill_dir(root)["findings"] assert any(finding["rule_id"] == "secret-cloud-token" and finding["severity"] == "CRITICAL" for finding in findings) @pytest.mark.parametrize("value", [{"mean": 3800.5, "stddev": 400, "min": 3200, "max": 4100}, "+1700", "-12.5"]) def test_benchmark_token_summaries_are_measurements(tmp_path: Path, value) -> None: root = _write_package(tmp_path, "benchmark.md", json.dumps({"tokens": value})) assert _secret_findings(root) == []