From 3c0a45ad772cdba388009b8d5ecad5e48cd22429 Mon Sep 17 00:00:00 2001 From: Aari Date: Tue, 21 Jul 2026 23:41:07 +0800 Subject: [PATCH] fix(skills): inject Langfuse metadata into the standalone skill scan (#4321) --- .../deerflow/skills/security_scanner.py | 24 ++++- backend/tests/test_security_scanner.py | 89 +++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/backend/packages/harness/deerflow/skills/security_scanner.py b/backend/packages/harness/deerflow/skills/security_scanner.py index 5534250ef..b3bb9f30c 100644 --- a/backend/packages/harness/deerflow/skills/security_scanner.py +++ b/backend/packages/harness/deerflow/skills/security_scanner.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import logging +import os import re from dataclasses import dataclass from typing import Any @@ -11,7 +12,9 @@ from typing import Any from deerflow.config import get_app_config from deerflow.config.app_config import AppConfig from deerflow.models import create_chat_model +from deerflow.runtime.user_context import get_effective_user_id from deerflow.skills.types import SKILL_MD_FILE +from deerflow.tracing import inject_langfuse_metadata logger = logging.getLogger(__name__) @@ -125,12 +128,31 @@ async def scan_skill_content( model_name = config.skill_evolution.moderation_model_name model_kwargs = {"thinking_enabled": False, "app_config": config, "attach_tracing": attach_tracing} model = create_chat_model(name=model_name, **model_kwargs) if model_name else create_chat_model(**model_kwargs) + invoke_config: dict[str, Any] = {"run_name": "security_agent"} + if attach_tracing: + # Standalone callers own the trace root, so they must inject their own + # Langfuse attribution -- the other half of the standalone pattern that + # already attaches model-level callbacks here (attach_tracing default), + # mirroring oneshot_llm.run_oneshot_llm / MemoryUpdater / the goal + # evaluator (see the Tracing System INVARIANT in backend/AGENTS.md). + # In-graph callers pass attach_tracing=False: the graph root already + # lifts session/user attribution, so injecting here is inert at best + # and diverges from that documented split. thread_id=None because the + # skill-moderation call is not thread-scoped (same as oneshot_llm). + inject_langfuse_metadata( + invoke_config, + thread_id=None, + user_id=get_effective_user_id(), + assistant_id="security_agent", + model_name=model_name, + environment=os.environ.get("DEER_FLOW_ENV") or os.environ.get("ENVIRONMENT"), + ) response = await model.ainvoke( [ {"role": "system", "content": rubric}, {"role": "user", "content": prompt}, ], - config={"run_name": "security_agent"}, + config=invoke_config, ) model_responded = True raw = str(getattr(response, "content", "") or "") diff --git a/backend/tests/test_security_scanner.py b/backend/tests/test_security_scanner.py index 1471c4533..542ed3beb 100644 --- a/backend/tests/test_security_scanner.py +++ b/backend/tests/test_security_scanner.py @@ -27,6 +27,40 @@ def _make_env(monkeypatch, response_content): return model +def _make_traced_env(monkeypatch, *, model_name, response_content='{"decision":"allow","reason":"ok"}'): + """Like ``_make_env`` but with a concrete moderation model name and a known + effective user, so Langfuse trace metadata (model tag + user_id) is assertable. + """ + config = SimpleNamespace(skill_evolution=SimpleNamespace(moderation_model_name=model_name)) + fake_response = SimpleNamespace(content=response_content) + + class FakeModel: + async def ainvoke(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + return fake_response + + model = FakeModel() + + def _fake_create_chat_model(**kwargs): + model.create_kwargs = kwargs + return model + + monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config) + monkeypatch.setattr("deerflow.skills.security_scanner.create_chat_model", _fake_create_chat_model) + monkeypatch.setattr("deerflow.skills.security_scanner.get_effective_user_id", lambda: "scanner-user") + return model + + +def _enable_langfuse_env(monkeypatch): + for name in ("LANGFUSE_TRACING", "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + monkeypatch.setenv("DEER_FLOW_ENV", "production") + + SKILL_CONTENT = "---\nname: demo-skill\ndescription: demo\n---\n" @@ -180,6 +214,61 @@ async def test_scan_skill_content_attaches_model_tracing_by_default(monkeypatch) assert model.create_kwargs["attach_tracing"] is True +@pytest.mark.anyio +async def test_scan_skill_content_injects_langfuse_metadata_when_standalone(monkeypatch): + """Standalone scans (Gateway routes, installer) own the trace root, so they must + inject Langfuse attribution themselves -- the other half of the standalone pattern + that already attaches model-level callbacks here, mirroring oneshot_llm / the goal + evaluator / MemoryUpdater (Tracing System INVARIANT in backend/AGENTS.md). Without + it the skill-moderation trace has no user/session/name attribution (the #4252 + follow-up gap). + """ + from deerflow.config.tracing_config import reset_tracing_config + + _enable_langfuse_env(monkeypatch) + reset_tracing_config() + model = _make_traced_env(monkeypatch, model_name="moderation-model") + try: + result = await scan_skill_content(SKILL_CONTENT, executable=False) + finally: + reset_tracing_config() + + assert result.decision == "allow" + config = model.kwargs["config"] + assert config["run_name"] == "security_agent" + metadata = config.get("metadata") or {} + assert metadata.get("langfuse_user_id") == "scanner-user" + assert metadata.get("langfuse_trace_name") == "security_agent" + # Skill moderation is not thread-scoped, so session_id stays None (matches + # oneshot_llm's thread_id=None); the key must still be present for the handler. + assert "langfuse_session_id" in metadata + assert metadata["langfuse_session_id"] is None + tags = metadata.get("langfuse_tags") or [] + assert "model:moderation-model" in tags + assert "env:production" in tags + + +@pytest.mark.anyio +async def test_scan_skill_content_omits_langfuse_metadata_when_in_graph(monkeypatch): + """In-graph scans pass attach_tracing=False and inherit attribution from the graph + root, so the injection must be gated on attach_tracing. Anchors the narrowing + direction: an unconditional inject (dropping the guard) would double-attribute + against the root trace and turn this red, even though Langfuse is enabled. + """ + from deerflow.config.tracing_config import reset_tracing_config + + _enable_langfuse_env(monkeypatch) + reset_tracing_config() + model = _make_traced_env(monkeypatch, model_name="moderation-model") + try: + result = await scan_skill_content(SKILL_CONTENT, executable=False, attach_tracing=False) + finally: + reset_tracing_config() + + assert result.decision == "allow" + assert model.kwargs["config"] == {"run_name": "security_agent"} + + def _make_unavailable_env(monkeypatch, *, security_fail_closed): config = SimpleNamespace( skill_evolution=SimpleNamespace(