diff --git a/CHANGELOG.md b/CHANGELOG.md index 254b20cc2f..172b3ce6c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,15 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **scheduler:** Scheduled-task IM notices no longer forward a credential that + carries no secret-like key. A URL's user info + (`postgres://admin:hunter2@host`) and an `Authorization` or `Cookie` header + value (`Authorization: Bearer ...`) used to reach the platform verbatim; both + now read `[redacted]`, and the scheme with the rest of the sentence stays + readable. A URL without credentials is unchanged. A `Cookie` header or a + `Digest` credential is replaced through the end of the line, so text that + follows it on the same line is removed as well. ([#8250]) + - **frontend:** Cron presets in the task form now require minute 0-59, hour 0-23 and day of month 1-31, so an out-of-range expression such as `0 24 * * *` stays a custom schedule and keeps its original text. Previously, @@ -16441,3 +16450,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#8139]: https://github.com/bytedance/deer-flow/pull/8139 [#8151]: https://github.com/bytedance/deer-flow/pull/8151 [#8174]: https://github.com/bytedance/deer-flow/pull/8174 +[#8250]: https://github.com/bytedance/deer-flow/pull/8250 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 0749d5a0c0..1d67ea2bc4 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -33,6 +33,12 @@ ### 修复 +- **调度器:** 定时任务发往 IM 的通知不再把不带密钥词形态的凭证原样发出。URL 中的账号信息 + (`postgres://admin:hunter2@host`)以及 `Authorization`、`Cookie` 请求头的值 + (`Authorization: Bearer ...`)此前会完整进入消息;现在替换为 `[redacted]`,协议名和句子 + 其余部分保持可读,不含凭证的 URL 不受影响。`Cookie` 请求头和 `Digest` 凭据会替换到行尾, + 同一行后面紧跟的文字也会一并去掉。([#8250]) + - **前端:** 任务表单的 Cron 预设现在要求分钟 0-59、小时 0-23、日期 1-31, 超出范围的表达式(例如 `0 24 * * *`)会保持为自定义并保留原始文本。此前它会被识别为预设, 在序列化以供提交时可能被限制到合法范围内的另一个时间。使用非法表达式创建或复制任务, @@ -12918,3 +12924,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写,核心包含子 [#8126]: https://github.com/bytedance/deer-flow/pull/8126 [#8139]: https://github.com/bytedance/deer-flow/pull/8139 [#8174]: https://github.com/bytedance/deer-flow/pull/8174 +[#8250]: https://github.com/bytedance/deer-flow/pull/8250 diff --git a/backend/app/scheduler/notification_text.py b/backend/app/scheduler/notification_text.py index 1ff673f1f7..9dc7303937 100644 --- a/backend/app/scheduler/notification_text.py +++ b/backend/app/scheduler/notification_text.py @@ -39,21 +39,63 @@ _CONDITION_LIMIT = 200 _TITLE_LIMIT = 200 _EGRESS_REDACTION = "[redacted]" -# Best-effort scrub before run content leaves for a third-party IM platform. -# Patterns mirror the high-confidence secret detectors in skillscan. -_EGRESS_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = ( +# Whole blocks go before the line rules: a header rule would otherwise consume +# the opening delimiter, leaving the block pattern without a start marker and +# the key body behind it in plain text. +_EGRESS_BLOCK_PATTERNS: tuple[re.Pattern[str], ...] = ( re.compile( r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----", re.DOTALL, ), +) +# Best-effort scrub before run content leaves for a third-party IM platform. +# Patterns mirror the high-confidence secret detectors in skillscan. +_EGRESS_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = ( re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"), re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"), re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"), - re.compile(r"(?im)\b(token|password|passwd|api[_-]?key|secret|credential)s?\b\s*[:=]\s*[\"']?([^\"'\s#]+)"), + re.compile(r"(?im)\b(token|password|passwd|api[_-]?key|secret|credential|signature)s?\b\s*[:=]\s*[\"']?([^\"'\s#]+)"), ) +# The patterns above only reach a value that follows a secret-like key. Two +# credential carriers have no such key: a URL's user info +# (``postgres://user:pass@host``) and an ``Authorization``/``Cookie`` header +# value (``Authorization: Bearer ...``). Both are common in agent replies about +# deployments and API calls, and both would otherwise reach the IM platform +# verbatim. Only the credential itself is replaced, so the scheme and the rest +# of the line keep reading normally. +_URL_USERINFO_RE = re.compile(r"([a-zA-Z][\w+.-]*://)([^/?#\s@]+)@") +# The value may be quoted with the scheme outside the quotes, so both +# ``Authorization: Bearer x`` and ``Authorization: "Bearer x"`` are covered. +_AUTH_HEADER_RE = re.compile(r"(?i)(\bauthorization\b\s*[:=]\s*)(?:[\"']?(?:(bearer|basic|token|digest)\s+)?[\"']?)([^\s\"';]+)[\"']?") +# ``Digest`` carries the credential as a list of ``name=value`` fields +# (``username="…", nonce="…", response="…"``). Only the first field follows the +# scheme, so the token rule would leave the rest visible; the whole header value +# goes instead, the same way the cookie rule handles its pair list. +_DIGEST_HEADER_RE = re.compile(r"(?i)(\bauthorization\b\s*[:=]\s*)(?:[\"']?digest\s+)([^\n]+)") +# A cookie header carries one ``name=value`` pair per ``;``-separated segment, +# and only the first pair follows the header name, so the whole value goes. +_COOKIE_HEADER_RE = re.compile(r"(?i)(\bcookie\b\s*[:=]\s*)([^\n]+)") + + +def _redact_url_userinfo(match: re.Match[str]) -> str: + return f"{match.group(1)}{_EGRESS_REDACTION}@" + + +def _redact_auth_header(match: re.Match[str]) -> str: + scheme = match.group(2) + prefix = f"{match.group(1)}{scheme} " if scheme else match.group(1) + return f"{prefix}{_EGRESS_REDACTION}" + + +def _redact_digest_header(match: re.Match[str]) -> str: + return f"{match.group(1)}Digest {_EGRESS_REDACTION}" + + +def _redact_cookie_header(match: re.Match[str]) -> str: + return f"{match.group(1)}{_EGRESS_REDACTION}" def redact_egress_text(text: str) -> str: @@ -61,6 +103,12 @@ def redact_egress_text(text: str) -> str: if not text: return text redacted = text + for pattern in _EGRESS_BLOCK_PATTERNS: + redacted = pattern.sub(_EGRESS_REDACTION, redacted) + redacted = _URL_USERINFO_RE.sub(_redact_url_userinfo, redacted) + redacted = _DIGEST_HEADER_RE.sub(_redact_digest_header, redacted) + redacted = _AUTH_HEADER_RE.sub(_redact_auth_header, redacted) + redacted = _COOKIE_HEADER_RE.sub(_redact_cookie_header, redacted) for pattern in _EGRESS_SECRET_PATTERNS: if pattern.groups: redacted = pattern.sub(lambda match: f"{match.group(1)}={_EGRESS_REDACTION}", redacted) diff --git a/backend/tests/test_notification_text.py b/backend/tests/test_notification_text.py index c383a5fb4f..0d5a469eec 100644 --- a/backend/tests/test_notification_text.py +++ b/backend/tests/test_notification_text.py @@ -263,6 +263,90 @@ def test_secret_on_the_first_line_is_redacted(): assert "Result: token=[redacted]" in text +def test_redact_egress_text_scrubs_url_user_info(): + redacted = redact_egress_text("deployed with postgres://admin:hunter2@db.internal:5432/deerflow") + + assert "hunter2" not in redacted + assert redacted == "deployed with postgres://[redacted]@db.internal:5432/deerflow" + + +def test_redact_egress_text_leaves_a_url_without_credentials_alone(): + text = "opened http://127.0.0.1:2026/workspace?tab=runs and ssh://github.com/bytedance/deer-flow.git" + + assert redact_egress_text(text) == text + + +@pytest.mark.parametrize( + ("header", "secret"), + [ + ("Authorization: Bearer 1234567890abcdef", "1234567890abcdef"), + ("authorization=Basic dXNlcjpwYXNzd29yZA==", "dXNlcjpwYXNzd29yZA=="), + ("Cookie: session=8f14e45fceea167a5a36dedd4bea2543", "8f14e45fceea167a5a36dedd4bea2543"), + # Every pair of a canonical multi-cookie header, not only the first one. + ("Cookie: theme=dark; session=8f14e45fceea167a5a36dedd4bea2543", "8f14e45fceea167a5a36dedd4bea2543"), + # A quoted value keeps the scheme outside the quotes. + ('Authorization: "Bearer 8f14e45fceea167a5a36dedd4bea2543"', "8f14e45fceea167a5a36dedd4bea2543"), + ("Authorization: 'Bearer 8f14e45fceea167a5a36dedd4bea2543'", "8f14e45fceea167a5a36dedd4bea2543"), + # A comma joins several tokens into one header value. + ("Authorization: Bearer aaa111,bbb222", "bbb222"), + ], +) +def test_redact_egress_text_scrubs_header_credentials(header: str, secret: str): + redacted = redact_egress_text(f"sent the request with {header} to the service") + + assert secret not in redacted + assert "[redacted]" in redacted + + +def test_redact_egress_text_scrubs_the_aws_signature_field(): + # SigV4 keeps the signature as a `name=value` field after the scheme, so the + # header rule alone only redacts the scheme token. + signature = "8f14e45fceea167a5a36dedd4bea2543f7a1b2c3d4e5f60718293a4b5c6d7e8" + header = f"Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20261115/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature={signature}" + + redacted = redact_egress_text(f"sent the request with {header} to the bucket") + + assert signature not in redacted + assert "[redacted]" in redacted + + +def test_redact_egress_text_scrubs_the_whole_digest_credential(): + # Digest writes every credential as its own `name=value` field after the + # scheme, so only redacting the first field leaves the nonce and response + # visible. + nonce = "8f14e45fceea167a5a36dedd4bea2543" + response = "c0ffee1234567890abcdef0123456789" + header = f'Authorization: Digest username="someone", realm="api", nonce="{nonce}", uri="/v1/export", qop=auth, response="{response}"' + + redacted = redact_egress_text(f"sent {header} to the service") + + assert nonce not in redacted + assert response not in redacted + # The Digest rule runs to the end of the line, so the prose after the header + # goes with it; the guide and changelog describe that. + assert redacted == "sent Authorization: Digest [redacted]" + + +def test_redact_egress_text_scrubs_a_pem_block_behind_a_header_name(): + # A header rule that runs first would consume the `BEGIN` line and leave the + # block rule without a start marker, so the body would survive. + pem_body = "ABCDEFSECRETKEYBODY" + pem = "-----BEGIN RSA PRIVATE KEY-----\n" + pem_body + "\n-----END RSA PRIVATE KEY-----" + + redacted = redact_egress_text(f"Cookie: {pem}\nafter the key") + + assert pem_body not in redacted + assert "PRIVATE KEY" not in redacted + assert "after the key" in redacted + + +def test_result_summary_never_forwards_url_credentials(): + text = render_notification_text(notice("run_completed", run_status="success", result_summary="Connected to postgres://admin:hunter2@db.internal:5432/deerflow")) + + assert "hunter2" not in text + assert "postgres://[redacted]@db.internal:5432/deerflow" in text + + def test_stop_condition_is_redacted_and_truncated(): secret = "sk-" + ("S" * 40) condition = f"stop when {secret} rotates " + "z" * 400 diff --git a/docs/guide/scheduled-tasks.md b/docs/guide/scheduled-tasks.md index 6612e4d601..d41aba4f2f 100644 --- a/docs/guide/scheduled-tasks.md +++ b/docs/guide/scheduled-tasks.md @@ -34,6 +34,8 @@ Current MVP capabilities: Derived run summaries omit leading inline `` reasoning before limiting the text, so a long thought does not replace the actual answer in previews or scheduled notices. Original response history and literal tags in the answer are unchanged; previously stored run summaries are not rewritten. +Text that leaves for an IM platform through a notice is scrubbed first: known token shapes, key assignments, private-key blocks, a URL's embedded user info, and `Authorization` or `Cookie` header values are replaced with `[redacted]` in the task title, stop condition, and result summary. The scrub is best effort. Most rules keep the surrounding sentence, but a `Cookie` header and a `Digest` credential are replaced through the end of the line, so text that follows them on the same line is removed too. Check a notice before forwarding it somewhere else. + **Filter execution history through the API** To inspect failures without downloading every successful occurrence, authenticated clients with `threads:read` can request `GET /api/scheduled-tasks/{task_id}/runs?status=failed&limit=50&offset=0` for an owned task. The optional `status` accepts `queued`, `launching`, `running`, `success`, `failed`, `skipped`, `interrupted`, or `unmet`; these are occurrence statuses, so task statuses such as `completed` are invalid (422).