From e33024446ad6e2519c49add59bf94ab98bd3a386 Mon Sep 17 00:00:00 2001 From: yetuge <182123929+yetuge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:07:11 +0800 Subject: [PATCH] fix(crawl4ai): report a format error for non-object 200 payloads (#7065) Crawl4AI's client assumed a JSON 200 was an object and indexed it. A reverse proxy, a version-skewed server or an error document delivered with HTTP 200 can return a list or a bare scalar instead, and then: - `data.get(...)` raised "'list' object has no attribute 'get'", and the generic handler echoed that Python text to the agent; - a non-string `markdown` field reached `.strip()` and raised the same way one level in (["# heading"], 123, {"text": ...}). Both now report "Crawl4AI returned an unexpected response format" and log the unexpected type. This matches the sibling providers' contract -- tavily's `_response_objects` returns a format error for a non-object payload -- and the same containment the search providers already apply. Measured: 8 new parameterised cases fail on the base revision (5 for the payload, 3 for the markdown field); 39 tests in the file, 122 across the crawl4ai/browserless/sanitization neighbourhood, ruff check/format clean. Each guard was mutation-checked separately. --- .../community/crawl4ai/crawl4ai_client.py | 12 ++++ backend/tests/test_crawl4ai_tools.py | 61 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py index 0a97de809a..eaa68e971f 100644 --- a/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py +++ b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py @@ -44,10 +44,22 @@ class Crawl4AiClient: content_type = resp.headers.get("content-type", "unknown") return f"Error: Crawl4AI returned a non-JSON 200 response (content-type: {content_type}): {resp.text[:200]}" + # A JSON 200 that is not an object is a format error, not a + # success: indexing it raised "'list' object has no attribute + # 'get'" and the generic handler below echoed that Python text + # to the agent. Sibling providers (tavily's _response_objects) + # report a format error instead. + if not isinstance(data, dict): + logger.error("Crawl4AI returned an unexpected payload type: %s", type(data).__name__) + return "Error: Crawl4AI returned an unexpected response format" + if not data.get("success", False): return f"Error: Crawl4AI reported failure for {url}" markdown = data.get("markdown") or "" + if not isinstance(markdown, str): + logger.error("Crawl4AI returned an unexpected 'markdown' payload type: %s", type(markdown).__name__) + return "Error: Crawl4AI returned an unexpected response format" if not markdown.strip(): return "Error: Crawl4AI returned empty markdown" diff --git a/backend/tests/test_crawl4ai_tools.py b/backend/tests/test_crawl4ai_tools.py index 2e02422a32..d700eea5c5 100644 --- a/backend/tests/test_crawl4ai_tools.py +++ b/backend/tests/test_crawl4ai_tools.py @@ -168,6 +168,67 @@ class TestCrawl4AiClient: assert result.startswith("Error: Crawl4AI returned a non-JSON 200 response") assert "text/html" in result + @pytest.mark.parametrize( + "payload", + [[{"markdown": "# ok"}], "maintenance window", 42, None, True], + ids=["payload-is-a-list", "payload-is-a-string", "payload-is-an-int", "payload-is-null", "payload-is-a-bool"], + ) + async def test_fetch_markdown_non_object_200_reports_a_format_error(self, payload): + """A JSON 200 that is not an object is a format error, not a Python traceback. + + Crawl4AI answers with a JSON object on success. A reverse proxy, a + version-skewed server or an error document delivered with HTTP 200 can + return a list or a bare scalar instead; indexing it directly raised + ``'list' object has no attribute 'get'`` and the generic handler echoed + that Python text to the agent. Sibling providers (tavily's + ``_response_objects``) report a format error instead. + """ + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = payload + mock_resp.headers = {"content-type": "application/json"} + mock_resp.text = json.dumps(payload)[:200] + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + + assert result.startswith("Error: Crawl4AI returned an unexpected response format"), result + assert "object has no attribute" not in result + + @pytest.mark.parametrize( + "markdown", + [["# heading"], 123, {"text": "# heading"}], + ids=["markdown-is-a-list", "markdown-is-an-int", "markdown-is-an-object"], + ) + async def test_fetch_markdown_non_string_payload_reports_a_format_error(self, markdown): + """A non-string ``markdown`` field is the same format error, one level in. + + ``data.get("markdown") or ""`` passes a list/dict/int straight through to + ``.strip()``, which raised ``'list' object has no attribute 'strip'`` -- + the container guard above does not cover this field. + """ + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"success": True, "markdown": markdown} + mock_resp.headers = {"content-type": "application/json"} + mock_resp.text = "{}" + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + + assert result.startswith("Error: Crawl4AI returned an unexpected response format"), result + assert "object has no attribute" not in result + @pytest.mark.asyncio class TestCrawl4AiTools: