fix(infoquest): reject invalid extracted fetch content (#6760)

* fix(infoquest): reject invalid extracted fetch content

* fix(infoquest): address fetch response review feedback

Preserve reader_result precedence so malformed primary content is reported instead of silently masked by a secondary field. Include request URLs in invalid/empty content diagnostics and reject non-object JSON before field selection.

* test(infoquest): respect URL redaction in fetch diagnostics

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

backend/packages/harness/deerflow/community/infoquest/infoquest_client.py

@@ -97,17 +97,32 @@ class InfoQuestClient:
             # Try to parse response as JSON and extract reader_result
             try:
                 response_data = json.loads(response.text)
+                if not isinstance(response_data, dict):
+                    logger.warning("InfoQuest returned unexpected response type: %s for URL: %s", type(response_data).__name__, url)
+                    return "Error: InfoQuest returned an unexpected response format"
                 # Extract reader_result if it exists
+                # Preserve primary-field precedence: a malformed primary result must
+                # remain visible rather than being masked by secondary content.
                 if "reader_result" in response_data:
                     logger.debug("Successfully extracted reader_result from JSON response")
-                    return response_data["reader_result"]
+                    content_field = "reader_result"
                 elif "content" in response_data:
                     # Fallback to content field if reader_result is not available
-                    logger.debug("reader_result missing in JSON response, falling back to content field: %s", response_data["content"])
-                    return response_data["content"]
+                    logger.debug("reader_result missing in JSON response, falling back to content field")
+                    content_field = "content"
                 else:
                     # If neither field exists, return the original response
                     logger.warning("Neither reader_result nor content field found in JSON response")
+                    content_field = None
+                if content_field is not None:
+                    content = response_data[content_field]
+                    if not isinstance(content, str):
+                        logger.warning("InfoQuest returned unexpected %s type: %s for URL: %s", content_field, type(content).__name__, url)
+                        return "Error: InfoQuest returned an unexpected response format"
+                    if not content.strip():
+                        logger.debug("InfoQuest returned empty %s for URL: %s", content_field, url)
+                        return "Error: no result found"
+                    return content
             except json.JSONDecodeError:
                 # If response is not JSON, return the original text
                 logger.debug("Response is not in JSON format, returning as-is")

backend/tests/AGENTS.md

@@ -10,6 +10,11 @@ ownership and capture cleanup. Retain the real XLSX conversion coverage.
 
 Backend tests must preserve the runtime invariants they exercise without changing production execution topology.
 
+InfoQuest fetch log assertions install the real `UrlRedactionFilter` on the
+capture handler with fixture-scoped cleanup. Expect scheme/host context with
+redacted paths, so isolated runs and full-suite logging configuration agree;
+keep payload omission and tool-result assertions intact.
+
 `.github/workflows/backend-windows-live-policy.yml` runs the child-env contract and all
 four live-test collection policy cases on Windows/Python 3.12, weekly, manually,
 and for relevant PR changes. These children only collect live tests. Keep the

backend/tests/test_infoquest_fetch_content.py

@@ -0,0 +1,138 @@
+"""Exercise InfoQuest fetch decoding through the real client and public tool."""
+
+import json
+import logging
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+from deerflow.community.infoquest import infoquest_client, tools
+from deerflow.logging_config import UrlRedactionFilter
+
+_URL = "https://example.com/article"
+_LOG_URL = "https://example.com/<redacted>"
+_HTML = "<html><head><title>Example</title></head><body><p>Example article content.</p></body></html>"
+
+
+@pytest.fixture(autouse=True)
+def redact_captured_urls(caplog):
+    # Exercise the production log policy even when this module runs in isolation.
+    redaction_filter = UrlRedactionFilter()
+    caplog.handler.addFilter(redaction_filter)
+    try:
+        yield
+    finally:
+        caplog.handler.removeFilter(redaction_filter)
+
+
+@pytest.fixture
+def responses(monkeypatch):
+    queued = []
+    requests = []
+    clients = []
+
+    def respond(request):
+        requests.append(request)
+        return queued.pop(0)
+
+    def client_factory(**kwargs):
+        client = httpx.AsyncClient(transport=httpx.MockTransport(respond), **kwargs)
+        clients.append(client)
+        return client
+
+    # Patch only this provider's module reference, not the shared HTTPX class.
+    monkeypatch.setattr(infoquest_client, "httpx", SimpleNamespace(AsyncClient=client_factory))
+    monkeypatch.setattr(tools, "get_app_config", lambda: SimpleNamespace(get_tool_config=lambda _: None))
+    monkeypatch.setenv("INFOQUEST_API_KEY", "synthetic-test-key")
+    yield queued, requests
+    assert all(client.is_closed for client in clients)
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize("field", ["reader_result", "content"])
+@pytest.mark.parametrize("value", [None, False, 0, [], {}, ["provider-payload"], {"text": "provider-payload"}])
+async def test_fetch_tool_contains_non_text_content(responses, caplog, field, value):
+    queued, requests = responses
+    queued.append(httpx.Response(200, json={field: value}))
+
+    result = await tools.web_fetch_tool.ainvoke({"url": _URL})
+
+    assert result == "Error: InfoQuest returned an unexpected response format"
+    assert len(requests) == 1
+    assert json.loads(requests[0].content) == {"url": _URL, "format": "HTML"}
+    assert f"InfoQuest returned unexpected {field} type: {type(value).__name__} for URL: {_LOG_URL}" in caplog.text
+    assert _URL not in caplog.text
+    assert "provider-payload" not in caplog.text
+    assert "synthetic-test-key" not in caplog.text
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize("field", ["reader_result", "content"])
+@pytest.mark.parametrize("value", ["", " \t\n"])
+async def test_fetch_tool_reports_empty_extracted_content(responses, caplog, field, value):
+    caplog.set_level(logging.DEBUG, logger=infoquest_client.__name__)
+    queued, _ = responses
+    queued.append(httpx.Response(200, json={field: value}))
+
+    assert await tools.web_fetch_tool.ainvoke({"url": _URL}) == "Error: no result found"
+    assert f"InfoQuest returned empty {field} for URL: {_LOG_URL}" in caplog.text
+    assert _URL not in caplog.text
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize("payload", [None, False, 0, 1.5, "provider-payload", "reader_result", [], ["content"]])
+async def test_fetch_tool_rejects_non_object_json(responses, caplog, payload):
+    queued, requests = responses
+    # Explicit serialization is needed because httpx treats json=None as no body.
+    queued.append(httpx.Response(200, text=json.dumps(payload)))
+
+    assert await tools.web_fetch_tool.ainvoke({"url": _URL}) == "Error: InfoQuest returned an unexpected response format"
+    assert len(requests) == 1
+    assert f"InfoQuest returned unexpected response type: {type(payload).__name__} for URL: {_LOG_URL}" in caplog.text
+    assert _URL not in caplog.text
+    assert "provider-payload" not in caplog.text
+    assert "synthetic-test-key" not in caplog.text
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize(
+    "payload",
+    [{"reader_result": _HTML}, {"content": _HTML}, {"reader_result": _HTML, "content": "secondary"}],
+)
+async def test_fetch_preserves_text_and_field_precedence(responses, payload):
+    queued, _ = responses
+    queued.append(httpx.Response(200, json=payload))
+
+    assert await infoquest_client.InfoQuestClient().fetch(_URL) == _HTML
+
+
+@pytest.mark.anyio
+async def test_fetch_preserves_raw_html_and_unknown_json(responses):
+    queued, _ = responses
+    queued.extend([httpx.Response(200, text=_HTML), httpx.Response(200, text='{"other": "value"}')])
+    client = infoquest_client.InfoQuestClient()
+
+    assert await client.fetch(_URL) == _HTML
+    assert await client.fetch(_URL) == '{"other": "value"}'
+
+
+@pytest.mark.anyio
+async def test_fetch_does_not_hide_invalid_primary_content_with_secondary(responses):
+    queued, _ = responses
+    queued.append(httpx.Response(200, json={"reader_result": None, "content": _HTML}))
+
+    assert await tools.web_fetch_tool.ainvoke({"url": _URL}) == "Error: InfoQuest returned an unexpected response format"
+
+
+@pytest.mark.anyio
+async def test_fetch_tool_recovers_on_the_next_request(responses):
+    queued, requests = responses
+    queued.extend([httpx.Response(200, json={"reader_result": None}), httpx.Response(200, json={"content": _HTML})])
+
+    assert await tools.web_fetch_tool.ainvoke({"url": _URL}) == "Error: InfoQuest returned an unexpected response format"
+    result = await tools.web_fetch_tool.ainvoke({"url": _URL + "/next"})
+
+    assert result.startswith("# Example\n")
+    assert "Example article content." in result
+    assert [json.loads(request.content)["url"] for request in requests] == [_URL, _URL + "/next"]