github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-10-27 21:50:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(skills): recognize HTTP scheme casing consistently (#6355)

Browse Source 
* fix(skills): recognize HTTP scheme casing consistently

* test(skills): separate IPv6 endpoints from HTTP schemes

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
Daoyuan Li 2026-10-06 00:56:53 -07:00 committed by GitHub
parent ca1cae6185
commit 534142dfed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 31 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -1324,7 +1324,7 @@ Skill installs and agent-managed skill edits run through **SkillScan**, a native
Windows scripts (`.bat`, `.cmd`, `.ps1`, `.psm1`, `.js`, `.jse`, `.vbs`, `.vbe`, `.wsf`), HTML applications (`.hta`), and scriptlets (`.sct`) count as code even outside `scripts/`, regardless of filename case. They receive both SkillScan analysis and the installer's executable-code policy.
SkillScan warns about remote downloads piped into common shells, including sudo, interpreter paths, and shell line continuations. Pipes to non-shell tools such as `jq` and `tee` do not trigger this warning.
SkillScan treats HTTP hostnames case-insensitively and recognizes bracketed IPv6
SkillScan treats HTTP/HTTPS scheme spellings and HTTP hostnames case-insensitively and recognizes bracketed IPv6
loopback (`[::1]`) URLs as local. External IPv6 endpoints still trigger network findings,
and cloud-metadata hostname detection is case-insensitive.

2
backend/packages/harness/deerflow/skills/AGENTS.md
View File

File diff suppressed because one or more lines are too long

10
backend/packages/harness/deerflow/skills/skillscan/orchestrator.py
View File

@ -135,8 +135,8 @@ _SECRET_TOKEN_PATTERNS = tuple(
)
)
_SENSITIVE_PATH_RE = re.compile(r"(~/.ssh|/etc/passwd|/etc/shadow|/var/run/docker\.sock|docker\.sock|169\.254\.169\.254)")
_EXTERNAL_HTTP_RE = re.compile(r"http://(?:[^/?#\s)'\"<>]*@)?(\[[0-9A-Fa-f:.]+\]|[A-Za-z0-9.-]+)(?::\d+)?(?:/|\b|(?=$|[\s)'\"<>?#]))")
_URL_RE = re.compile(r"https?://[^\s)'\"<>]+")
_EXTERNAL_HTTP_RE = re.compile(r"(?i:http)://(?:[^/?#\s)'\"<>]*@)?(\[[0-9A-Fa-f:.]+\]|[A-Za-z0-9.-]+)(?::\d+)?(?:/|\b|(?=$|[\s)'\"<>?#]))")
_URL_RE = re.compile(r"(?i:https?)://[^\s)'\"<>]+")
_LOCAL_HTTP_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}
# `rm` with a recursive flag (any order/combination, optional --no-preserve-root)
# targeting the filesystem root, a wildcard, or a complete system-root directory.
@ -624,7 +624,7 @@ def _scan_declaration(rel_path: str, text: str) -> list[SecurityFinding]:
findings.append(_finding_from_match("declaration-sensitive-path", rel_path, text, match))
for match in _URL_RE.finditer(text):
if match.group(0).startswith("http://"):
if match.group(0)[:7].lower() == "http://":
host = _http_host(match.group(0))
if host and host not in _LOCAL_HTTP_HOSTS:
findings.append(_finding_from_match("declaration-external-endpoint", rel_path, text, match))
@ -1017,7 +1017,7 @@ def _looks_like_placeholder(value: str) -> bool:
def _http_host(url: str) -> str | None:
if not url.startswith(("http://", "https://")):
if not url[:8].lower().startswith(("http://", "https://")):
return None
try:
# Parse the authority so IPv6 brackets and userinfo cannot be mistaken
@ -1029,7 +1029,7 @@ def _http_host(url: str) -> str | None:
def _is_outbound_url(value: str) -> bool:
return bool(value.startswith(("http://", "https://")) and (_http_host(value) or "") not in _LOCAL_HTTP_HOSTS)
return bool(value[:8].lower().startswith(("http://", "https://")) and (_http_host(value) or "") not in _LOCAL_HTTP_HOSTS)
def _collect_python_aliases(tree: ast.AST) -> dict[str, str]:

40
backend/tests/test_skillscan_native.py
View File

@ -1905,6 +1905,9 @@ def test_shell_curl_without_pipe_finishes_on_repeated_backslash_text(tmp_path: P
"url, host",
[
("http://LOCALHOST:8080/api", "localhost"),
("HTTP://LOCALHOST:8080/api", "localhost"),
("HtTp://Example.COM/api", "example.com"),
("HTTPS://[::1]:8443/api", "::1"),
("http://[::1]/api", "::1"),
("https://[::1]:8443/api", "::1"),
("http://[2001:DB8::1]:8080/api", "2001:db8::1"),
@ -1934,9 +1937,10 @@ def test_uppercase_local_host_is_classified_local(tmp_path: Path) -> None:
@pytest.mark.parametrize("host, external", [("localhost", False), ("LOCALHOST", False), ("LocalHost", False), ("Example.COM", True), ("[::1]", False), ("[2001:DB8::1]", True), ("localhost@Example.COM", True)])
def test_declared_http_host_case_classification(tmp_path: Path, host: str, external: bool) -> None:
@pytest.mark.parametrize("scheme", ["http", "HTTP", "HtTp"])
def test_declared_http_host_case_classification(tmp_path: Path, host: str, external: bool, scheme: str) -> None:
skill_dir = tmp_path / "skill"
_write_skill(skill_dir, f"Endpoint: http://{host}:8080/api\n")
_write_skill(skill_dir, f"Endpoint: {scheme}://{host}:8080/api\n")
findings = scan_skill_dir(skill_dir)["findings"]
@ -1945,10 +1949,11 @@ def test_declared_http_host_case_classification(tmp_path: Path, host: str, exter
@pytest.mark.parametrize("host, external", [("localhost", False), ("LOCALHOST", False), ("LocalHost", False), ("Example.COM", True), ("[::1]", False), ("[2001:DB8::1]", True), ("[::1]@Example.COM", True)])
def test_sensitive_path_http_host_case_classification(tmp_path: Path, host: str, external: bool) -> None:
@pytest.mark.parametrize("scheme", ["http", "HTTP", "HtTp", "https", "HTTPS", "HtTpS"])
def test_sensitive_path_http_host_case_classification(tmp_path: Path, host: str, external: bool, scheme: str) -> None:
skill_dir = tmp_path / "skill"
_write_skill(skill_dir)
(skill_dir / "run.py").write_text(f'ENDPOINT = "http://{host}:8080/api"\nopen("/etc/passwd").read()\n', encoding="utf-8")
(skill_dir / "run.py").write_text(f'ENDPOINT = "{scheme}://{host}:8080/api"\nopen("/etc/passwd").read()\n', encoding="utf-8")
result = scan_skill_dir(skill_dir)
findings = result["findings"]
@ -1962,18 +1967,19 @@ def test_sensitive_path_http_host_case_classification(tmp_path: Path, host: str,
@pytest.mark.parametrize(
"url, local",
"endpoint, local",
[
("http://[::1]:8080/api", True),
("http://[::1]", True),
("http://[::1]?mode=local", True),
("http://[2001:DB8::1]:8080/api", False),
("http://[2001:db8::1]", False),
("[::1]:8080/api", True),
("[::1]", True),
("[::1]?mode=local", True),
("[2001:DB8::1]:8080/api", False),
("[2001:db8::1]", False),
],
)
def test_ipv6_cleartext_http_classification(tmp_path: Path, url: str, local: bool) -> None:
@pytest.mark.parametrize("scheme", ["http", "HTTP", "HtTp"])
def test_ipv6_cleartext_http_classification(tmp_path: Path, endpoint: str, local: bool, scheme: str) -> None:
skill_dir = tmp_path / "skill"
_write_skill(skill_dir, f"# Demo\nEndpoint: {url}\n")
_write_skill(skill_dir, f"# Demo\nEndpoint: {scheme}://{endpoint}\n")
findings = scan_skill_dir(skill_dir)["findings"]
@ -1985,10 +1991,11 @@ def test_ipv6_cleartext_http_classification(tmp_path: Path, url: str, local: boo
assert not [item for item in findings if item["rule_id"] == other_rule]
def test_malformed_ipv6_url_remains_outbound(tmp_path: Path) -> None:
@pytest.mark.parametrize("scheme", ["http", "HTTP", "HtTp", "https", "HTTPS"])
def test_malformed_ipv6_url_remains_outbound(tmp_path: Path, scheme: str) -> None:
skill_dir = tmp_path / "skill"
_write_skill(skill_dir)
(skill_dir / "run.py").write_text('ENDPOINT = "http://[::1/api"\nopen("/etc/passwd").read()\n', encoding="utf-8")
(skill_dir / "run.py").write_text(f'ENDPOINT = "{scheme}://[::1/api"\nopen("/etc/passwd").read()\n', encoding="utf-8")
result = scan_skill_dir(skill_dir)
@ -1997,9 +2004,10 @@ def test_malformed_ipv6_url_remains_outbound(tmp_path: Path) -> None:
assert result["scanner_errors"] == []
def test_cleartext_http_uses_host_after_userinfo_without_exposing_credentials(tmp_path: Path) -> None:
@pytest.mark.parametrize("scheme", ["http", "HTTP", "HtTp"])
def test_cleartext_http_uses_host_after_userinfo_without_exposing_credentials(tmp_path: Path, scheme: str) -> None:
skill_dir = tmp_path / "skill"
_write_skill(skill_dir, "Endpoint: http://localhost:private-value@Example.COM:8080/api\n")
_write_skill(skill_dir, f"Endpoint: {scheme}://localhost:private-value@Example.COM:8080/api\n")
findings = scan_skill_dir(skill_dir)["findings"]