From 192ea5155eca6f818315985de6db2259ef4c9cba Mon Sep 17 00:00:00 2001 From: Willem Jiang Date: Tue, 14 Jul 2026 11:57:48 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"fix(firecrawl):=20align=20tools=20wit?= =?UTF-8?q?h=20installed=20firecrawl-py=20v2=20SDK=20surfac=E2=80=A6"=20(#?= =?UTF-8?q?4165)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit b7b4b49bfea18171cf7ba5091ffd1f9a52dd43e8. --- .../deerflow/community/firecrawl/tools.py | 234 +-------- backend/tests/test_firecrawl_tools.py | 489 +----------------- 2 files changed, 11 insertions(+), 712 deletions(-) diff --git a/backend/packages/harness/deerflow/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py index 16fe1506e..86f44150a 100644 --- a/backend/packages/harness/deerflow/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -1,18 +1,17 @@ import json -from firecrawl import Firecrawl -from firecrawl.v2.types import ScrapeOptions +from firecrawl import FirecrawlApp from langchain.tools import tool from deerflow.config import get_app_config -def _get_firecrawl_client(tool_name: str = "web_search") -> Firecrawl: +def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp: config = get_app_config().get_tool_config(tool_name) api_key = None if config is not None and "api_key" in config.model_extra: api_key = config.model_extra.get("api_key") - return Firecrawl(api_key=api_key) # type: ignore[arg-type] + return FirecrawlApp(api_key=api_key) # type: ignore[arg-type] @tool("web_search", parse_docstring=True) @@ -59,15 +58,8 @@ def web_fetch_tool(url: str) -> str: url: The URL to fetch the contents of. """ try: - config = get_app_config().get_tool_config("web_fetch") - max_chars = 16384 - formats = ["markdown"] - if config is not None: - max_chars = config.model_extra.get("max_chars", max_chars) - formats = config.model_extra.get("formats", formats) - client = _get_firecrawl_client("web_fetch") - result = client.scrape(url, formats=formats) + result = client.scrape(url, formats=["markdown"]) markdown_content = result.markdown or "" metadata = result.metadata @@ -78,220 +70,4 @@ def web_fetch_tool(url: str) -> str: except Exception as e: return f"Error: {str(e)}" - return f"# {title}\n\n{markdown_content[:max_chars]}" - - -@tool("web_map", parse_docstring=True) -def web_map_tool(url: str) -> str: - """Discover URLs on a website. Acts like a sitemap generator — given a URL, - returns a list of URLs found on that site. - - Args: - url: The website URL to map (e.g. https://example.com). - """ - try: - config = get_app_config().get_tool_config("web_map") - limit = 100 - include_subdomains = True - ignore_sitemap = False - if config is not None: - limit = config.model_extra.get("limit", limit) - ignore_sitemap = config.model_extra.get("ignore_sitemap", ignore_sitemap) - include_subdomains = config.model_extra.get("include_subdomains", include_subdomains) - - client = _get_firecrawl_client("web_map") - result = client.map( - url, - limit=limit, - include_subdomains=include_subdomains, - sitemap="skip" if ignore_sitemap else None, - ) - - urls = getattr(result, "links", None) or [] - normalized = { - "urls": urls, - "total_count": len(urls), - } - return json.dumps(normalized, indent=2, ensure_ascii=False) - except Exception as e: - return f"Error: {str(e)}" - - -@tool("web_crawl", parse_docstring=True) -def web_crawl_tool(url: str) -> str: - """Crawl a website starting from a given URL. - - Args: - url: The starting URL to crawl from. - """ - try: - config = get_app_config().get_tool_config("web_crawl") - max_discovery_depth = 2 - limit = 10 - allow_subdomains = True - scrape_formats = ["markdown"] - if config is not None: - max_discovery_depth = config.model_extra.get("max_depth", max_discovery_depth) - limit = config.model_extra.get("limit", limit) - allow_subdomains = config.model_extra.get("allow_subdomains", allow_subdomains) - scrape_formats = config.model_extra.get("scrape_formats", scrape_formats) - - client = _get_firecrawl_client("web_crawl") - result = client.crawl( - url, - max_discovery_depth=max_discovery_depth, - limit=limit, - allow_subdomains=allow_subdomains, - scrape_options=ScrapeOptions(formats=scrape_formats), # type: ignore[arg-type] - ) - - pages = [] - for page in result.data or []: - page_url = "" - page_md = "" - if isinstance(page, dict): - page_url = page.get("url", "") or "" - page_md = page.get("markdown", "") or "" - else: - page_url = getattr(page, "url", None) or "" - page_md = getattr(page, "markdown", None) or "" - pages.append( - { - "url": page_url, - "markdown": page_md[:8192], - } - ) - - return json.dumps(pages, indent=2, ensure_ascii=False) - except Exception as e: - return f"Error: {str(e)}" - - -@tool("web_interact", parse_docstring=True) -def web_interact_tool(url: str, actions: str) -> str: - """Interact with a web page using browser automation. Opens a browser, performs - the described actions, and returns the final page content. - - Use this tool to fill forms, click buttons, navigate dynamic pages, or any - other browser-based interaction that requires simulating user behavior. - - ``actions`` must be a JSON array of action objects. Each object has a ``type`` - field and type-specific fields: - - - ``{"type": "wait", "milliseconds": 2000}`` — wait N ms - - ``{"type": "click", "selector": "#login-btn"}`` — click an element - - ``{"type": "write", "selector": "#search", "text": "hello"}`` — type text - - ``{"type": "press", "key": "Enter"}`` — press a key - - ``{"type": "scroll", "direction": "down", "amount": 300}`` — scroll - - ``{"type": "screenshot", "full_page": true}`` — take a screenshot - - ``{"type": "execute_javascript", "code": "..."}`` — run JS - - ``{"type": "scrape", "selector": "body"}`` — extract HTML from a selector - - Args: - url: The starting URL to open in the browser. - actions: JSON array of browser action objects to perform on the page. - """ - try: - config = get_app_config().get_tool_config("web_interact") - timeout = 30000 - wait_for = 2000 - formats = ["markdown"] - if config is not None: - timeout = config.model_extra.get("timeout", timeout) - wait_for = config.model_extra.get("wait_for", wait_for) - formats = config.model_extra.get("formats", formats) - - # Parse the action objects from JSON - action_dicts = json.loads(actions) - if not isinstance(action_dicts, list): - return "Error: actions must be a JSON array of action objects" - - # Build structured action objects - parsed_actions = [] - for act in action_dicts: - act_type = act.get("type", "") - if act_type == "wait": - parsed_actions.append({"type": "wait", "milliseconds": act.get("milliseconds", 1000)}) - elif act_type == "click": - parsed_actions.append({"type": "click", "selector": act["selector"]}) - elif act_type == "write": - parsed_actions.append({"type": "write", "selector": act["selector"], "text": act["text"]}) - elif act_type == "press": - parsed_actions.append({"type": "press", "key": act["key"]}) - elif act_type == "scroll": - parsed_actions.append({"type": "scroll", "direction": act.get("direction", "down"), "amount": act.get("amount", 300)}) - elif act_type == "screenshot": - parsed_actions.append({"type": "screenshot", "full_page": act.get("full_page", False)}) - elif act_type == "execute_javascript": - parsed_actions.append({"type": "executeJavascript", "code": act["code"]}) - elif act_type == "scrape": - parsed_actions.append({"type": "scrape", "selector": act.get("selector", "body")}) - else: - return f"Error: unknown action type '{act_type}'" - - client = _get_firecrawl_client("web_interact") - result = client.scrape( - url, - formats=formats, # type: ignore[arg-type] - actions=parsed_actions, # type: ignore[arg-type] - timeout=timeout, - wait_for=wait_for, - ) - - markdown_content = getattr(result, "markdown", None) or "" - if not markdown_content: - return "Error: No content returned" - - return markdown_content - except json.JSONDecodeError: - return "Error: actions must be valid JSON" - except Exception as e: - return f"Error: {str(e)}" - - -@tool("structured_extract", parse_docstring=True) -def structured_extract_tool( - urls: list[str], - prompt: str, - schema: dict | None = None, -) -> str: - """Extract structured data from web pages using an LLM-powered schema. - - Provide a list of URLs and a prompt describing what data to extract. Optionally - supply a JSON schema dict to enforce a specific output structure; if omitted, - Firecrawl will infer the schema from the prompt. - - Args: - urls: List of URLs to extract data from. - prompt: Natural-language description of the data to extract (e.g. "Extract product name, price, and rating for each product"). - schema: Optional JSON schema dict defining the expected output structure. - """ - try: - config = get_app_config().get_tool_config("structured_extract") - max_urls = 10 - if config is not None: - max_urls = config.model_extra.get("max_urls", max_urls) - - client = _get_firecrawl_client("structured_extract") - - trimmed_urls = urls[:max_urls] - kwargs = { - "urls": trimmed_urls, - "prompt": prompt, - } - if schema is not None: - kwargs["schema"] = schema - - result = client.extract(**kwargs) - - data = getattr(result, "data", None) - if data is None: - return json.dumps( - {"error": "No data returned from extraction", "success": False}, - indent=2, - ensure_ascii=False, - ) - - return json.dumps(data, indent=2, ensure_ascii=False) - except Exception as e: - return f"Error: {str(e)}" + return f"# {title}\n\n{markdown_content[:4096]}" diff --git a/backend/tests/test_firecrawl_tools.py b/backend/tests/test_firecrawl_tools.py index 520a38f12..fd61f817e 100644 --- a/backend/tests/test_firecrawl_tools.py +++ b/backend/tests/test_firecrawl_tools.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch class TestWebSearchTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") + @patch("deerflow.community.firecrawl.tools.FirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls): search_config = MagicMock() @@ -35,15 +35,11 @@ class TestWebSearchTool: class TestWebFetchTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") + @patch("deerflow.community.firecrawl.tools.FirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() - fetch_config.model_extra = { - "api_key": "firecrawl-fetch-key", - "max_chars": 50, - "formats": ["markdown", "html"], - } + fetch_config.model_extra = {"api_key": "firecrawl-fetch-key"} def get_tool_config(name): if name == "web_fetch": @@ -52,9 +48,8 @@ class TestWebFetchTool: mock_get_app_config.return_value.get_tool_config.side_effect = get_tool_config - long_content = "This is a much longer markdown content that exceeds fifty characters." mock_scrape_result = MagicMock() - mock_scrape_result.markdown = long_content + mock_scrape_result.markdown = "Fetched markdown" mock_scrape_result.metadata = MagicMock(title="Fetched Page") mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result @@ -62,482 +57,10 @@ class TestWebFetchTool: result = web_fetch_tool.invoke({"url": "https://example.com"}) - # Content should be truncated to max_chars (50) - expected = "# Fetched Page\n\n" + long_content[:50] - assert result == expected - assert len(result) == len("# Fetched Page\n\n") + 50 + assert result == "# Fetched Page\n\nFetched markdown" mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch") mock_firecrawl_cls.assert_called_once_with(api_key="firecrawl-fetch-key") mock_firecrawl_cls.return_value.scrape.assert_called_once_with( "https://example.com", - formats=["markdown", "html"], + formats=["markdown"], ) - - -class TestWebMapTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_map_success(self, mock_get_app_config, mock_firecrawl_cls): - """map returns a list of URLs — they should be serialized as JSON.""" - config = MagicMock() - config.model_extra = {"api_key": "map-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.links = [ - "https://example.com", - "https://example.com/about", - "https://example.com/contact", - ] - mock_firecrawl_cls.return_value.map.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_map_tool - - result = web_map_tool.invoke({"url": "https://example.com"}) - - parsed = json.loads(result) - assert parsed["urls"] == [ - "https://example.com", - "https://example.com/about", - "https://example.com/contact", - ] - assert parsed["total_count"] == 3 - mock_firecrawl_cls.return_value.map.assert_called_once_with( - "https://example.com", - limit=100, - include_subdomains=True, - sitemap=None, - ) - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_map_respects_config(self, mock_get_app_config, mock_firecrawl_cls): - """Config values for limit, ignore_sitemap, include_subdomains pass through to map.""" - config = MagicMock() - config.model_extra = { - "api_key": "cfg-key", - "limit": 42, - "ignore_sitemap": True, - "include_subdomains": False, - } - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.links = ["https://example.com/page"] - mock_firecrawl_cls.return_value.map.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_map_tool - - web_map_tool.invoke({"url": "https://example.com"}) - - mock_get_app_config.return_value.get_tool_config.assert_called_with("web_map") - mock_firecrawl_cls.return_value.map.assert_called_once_with( - "https://example.com", - limit=42, - include_subdomains=False, - sitemap="skip", - ) - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_map_empty_site(self, mock_get_app_config, mock_firecrawl_cls): - """Empty site — no URLs found — returns empty list with count 0.""" - config = MagicMock() - config.model_extra = {"api_key": "map-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.links = [] - mock_firecrawl_cls.return_value.map.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_map_tool - - result = web_map_tool.invoke({"url": "https://empty.example.com"}) - - parsed = json.loads(result) - assert parsed["urls"] == [] - assert parsed["total_count"] == 0 - - -class TestWebCrawlTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_crawl_success(self, mock_get_app_config, mock_firecrawl_cls): - """crawl returns a list of pages — they should be serialized as JSON with url + markdown.""" - config = MagicMock() - config.model_extra = {"api_key": "crawl-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_doc1 = MagicMock() - mock_doc1.url = "https://example.com" - mock_doc1.markdown = "Page 1 content" - mock_doc2 = MagicMock() - mock_doc2.url = "https://example.com/page2" - mock_doc2.markdown = "Page 2 content" - mock_result = MagicMock() - mock_result.data = [mock_doc1, mock_doc2] - mock_firecrawl_cls.return_value.crawl.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_crawl_tool - - result = web_crawl_tool.invoke({"url": "https://example.com"}) - - parsed = json.loads(result) - assert len(parsed) == 2 - assert parsed[0]["url"] == "https://example.com" - assert parsed[0]["markdown"] == "Page 1 content" - assert parsed[1]["url"] == "https://example.com/page2" - assert parsed[1]["markdown"] == "Page 2 content" - - @patch("deerflow.community.firecrawl.tools.ScrapeOptions") - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_crawl_respects_config(self, mock_get_app_config, mock_firecrawl_cls, mock_scrape_options): - """Config values for max_depth, limit, allow_subdomains, scrape_formats pass through to crawl.""" - config = MagicMock() - config.model_extra = { - "api_key": "cfg-key", - "max_depth": 3, - "limit": 20, - "allow_subdomains": False, - "scrape_formats": ["html"], - } - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.data = [] - mock_firecrawl_cls.return_value.crawl.return_value = mock_result - mock_scrape_options.return_value = "scrape_opts_mock" - - from deerflow.community.firecrawl.tools import web_crawl_tool - - web_crawl_tool.invoke({"url": "https://example.com"}) - - mock_get_app_config.return_value.get_tool_config.assert_called_with("web_crawl") - mock_scrape_options.assert_called_once_with(formats=["html"]) - mock_firecrawl_cls.return_value.crawl.assert_called_once_with( - "https://example.com", - max_discovery_depth=3, - limit=20, - allow_subdomains=False, - scrape_options="scrape_opts_mock", - ) - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_crawl_error(self, mock_get_app_config, mock_firecrawl_cls): - """Exceptions from crawl are caught and returned as 'Error: ...' string.""" - config = MagicMock() - config.model_extra = {"api_key": "crawl-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_firecrawl_cls.return_value.crawl.side_effect = RuntimeError("Crawl failed") - - from deerflow.community.firecrawl.tools import web_crawl_tool - - result = web_crawl_tool.invoke({"url": "https://example.com"}) - - assert result == "Error: Crawl failed" - - -class TestWebInteractTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_interact_success(self, mock_get_app_config, mock_firecrawl_cls): - """scrape() with browser actions returns markdown content — it should be passed through as-is.""" - config = MagicMock() - config.model_extra = {"api_key": "interact-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.markdown = "# Final Page\n\nContent after interaction." - mock_firecrawl_cls.return_value.scrape.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_interact_tool - - actions_json = json.dumps( - [ - {"type": "click", "selector": "#login-btn"}, - {"type": "wait", "milliseconds": 1000}, - ] - ) - result = web_interact_tool.invoke( - { - "url": "https://example.com", - "actions": actions_json, - } - ) - - assert result == "# Final Page\n\nContent after interaction." - mock_get_app_config.return_value.get_tool_config.assert_called_with("web_interact") - mock_firecrawl_cls.return_value.scrape.assert_called_once() - call_args, call_kwargs = mock_firecrawl_cls.return_value.scrape.call_args - assert call_args[0] == "https://example.com" - assert call_kwargs["formats"] == ["markdown"] - assert call_kwargs["timeout"] == 30000 - assert call_kwargs["wait_for"] == 2000 - # Verify actions were parsed into dicts - assert len(call_kwargs["actions"]) == 2 - assert call_kwargs["actions"][0] == {"type": "click", "selector": "#login-btn"} - assert call_kwargs["actions"][1] == {"type": "wait", "milliseconds": 1000} - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_interact_actions_pass_through(self, mock_get_app_config, mock_firecrawl_cls): - """The actions JSON should be parsed into action dicts and passed to scrape().""" - config = MagicMock() - config.model_extra = {"api_key": "interact-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_result = MagicMock() - mock_result.markdown = "Done." - mock_firecrawl_cls.return_value.scrape.return_value = mock_result - - from deerflow.community.firecrawl.tools import web_interact_tool - - actions_json = json.dumps( - [ - {"type": "write", "selector": "#search", "text": "hello"}, - {"type": "press", "key": "Enter"}, - ] - ) - web_interact_tool.invoke( - { - "url": "https://example.com", - "actions": actions_json, - } - ) - - call_kwargs = mock_firecrawl_cls.return_value.scrape.call_args.kwargs - assert call_kwargs["actions"] == [ - {"type": "write", "selector": "#search", "text": "hello"}, - {"type": "press", "key": "Enter"}, - ] - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_interact_error(self, mock_get_app_config, mock_firecrawl_cls): - """Exceptions from scrape() are caught and returned as 'Error: ...' string.""" - config = MagicMock() - config.model_extra = {"api_key": "interact-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_firecrawl_cls.return_value.scrape.side_effect = RuntimeError("Browser interaction timed out") - - from deerflow.community.firecrawl.tools import web_interact_tool - - actions_json = json.dumps([{"type": "click", "selector": "#btn"}]) - result = web_interact_tool.invoke( - { - "url": "https://example.com", - "actions": actions_json, - } - ) - - assert result == "Error: Browser interaction timed out" - - -class TestStructuredExtractTool: - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_extract_with_schema(self, mock_get_app_config, mock_firecrawl_cls): - """extract() with a schema returns the structured data as JSON.""" - config = MagicMock() - config.model_extra = {"api_key": "extract-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - extracted_data = { - "products": [ - {"name": "Widget", "price": 9.99, "rating": 4.5}, - {"name": "Gadget", "price": 19.99, "rating": 4.2}, - ] - } - mock_result = MagicMock() - mock_result.data = extracted_data - mock_firecrawl_cls.return_value.extract.return_value = mock_result - - from deerflow.community.firecrawl.tools import structured_extract_tool - - schema = { - "type": "object", - "properties": { - "products": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "price": {"type": "number"}, - "rating": {"type": "number"}, - }, - }, - } - }, - } - result = structured_extract_tool.invoke( - { - "urls": ["https://shop.example.com"], - "prompt": "Extract product name, price, and rating", - "schema": schema, - } - ) - - parsed = json.loads(result) - assert parsed == extracted_data - mock_get_app_config.return_value.get_tool_config.assert_called_with("structured_extract") - mock_firecrawl_cls.return_value.extract.assert_called_once_with( - urls=["https://shop.example.com"], - prompt="Extract product name, price, and rating", - schema=schema, - ) - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_extract_no_schema(self, mock_get_app_config, mock_firecrawl_cls): - """extract() without a schema (prompt-only) infers structure from the prompt.""" - config = MagicMock() - config.model_extra = {"api_key": "extract-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - extracted_data = {"title": "Example", "summary": "A test summary"} - mock_result = MagicMock() - mock_result.data = extracted_data - mock_firecrawl_cls.return_value.extract.return_value = mock_result - - from deerflow.community.firecrawl.tools import structured_extract_tool - - result = structured_extract_tool.invoke( - { - "urls": ["https://example.com"], - "prompt": "Extract the title and summary", - } - ) - - parsed = json.loads(result) - assert parsed == extracted_data - # Schema should NOT be passed when omitted - mock_firecrawl_cls.return_value.extract.assert_called_once_with( - urls=["https://example.com"], - prompt="Extract the title and summary", - ) - - @patch("deerflow.community.firecrawl.tools.Firecrawl") - @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_extract_error(self, mock_get_app_config, mock_firecrawl_cls): - """Exceptions from extract() are caught and returned as 'Error: ...' string.""" - config = MagicMock() - config.model_extra = {"api_key": "extract-key"} - mock_get_app_config.return_value.get_tool_config.return_value = config - - mock_firecrawl_cls.return_value.extract.side_effect = ValueError("Invalid URL format") - - from deerflow.community.firecrawl.tools import structured_extract_tool - - result = structured_extract_tool.invoke( - { - "urls": ["not-a-url"], - "prompt": "Extract data", - } - ) - - assert result == "Error: Invalid URL format" - - -class TestRealFirecrawlAppContract: - """Verify that the methods called by tools.py actually exist on the real - firecrawl-py FirecrawlApp with the parameter names we expect. - - This guards against SDK upgrades changing signatures without our tools - being updated. - """ - - @staticmethod - def test_client_methods_exist(): - """All methods used by tools.py must be present on a Firecrawl instance.""" - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - - expected_methods = ["map", "crawl", "scrape", "search", "extract"] - for name in expected_methods: - method = getattr(client, name, None) - assert method is not None, f"Firecrawl instance is missing '{name}' method — tools.py will fail at runtime" - - @staticmethod - def test_map_signature(): - """map() must accept url, limit, include_subdomains, sitemap.""" - import inspect - - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - sig = inspect.signature(client.map) - params = sig.parameters - - assert "url" in params, f"map() missing 'url' param; sig={sig}" - assert "limit" in params, f"map() missing 'limit' param; sig={sig}" - assert "include_subdomains" in params, f"map() missing 'include_subdomains' param; sig={sig}" - assert "sitemap" in params, f"map() missing 'sitemap' param; sig={sig}" - - @staticmethod - def test_crawl_signature(): - """crawl() must accept url, max_discovery_depth, limit, allow_subdomains, scrape_options.""" - import inspect - - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - sig = inspect.signature(client.crawl) - params = sig.parameters - - assert "url" in params, f"crawl() missing 'url' param; sig={sig}" - assert "max_discovery_depth" in params, f"crawl() missing 'max_discovery_depth' param; sig={sig}" - assert "limit" in params, f"crawl() missing 'limit' param; sig={sig}" - assert "allow_subdomains" in params, f"crawl() missing 'allow_subdomains' param; sig={sig}" - assert "scrape_options" in params, f"crawl() missing 'scrape_options' param; sig={sig}" - - @staticmethod - def test_scrape_signature(): - """scrape() must accept url, formats, actions, timeout, wait_for.""" - import inspect - - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - sig = inspect.signature(client.scrape) - params = sig.parameters - - assert "url" in params, f"scrape() missing 'url' param; sig={sig}" - assert "formats" in params, f"scrape() missing 'formats' param; sig={sig}" - assert "actions" in params, f"scrape() missing 'actions' param; sig={sig}" - assert "timeout" in params, f"scrape() missing 'timeout' param; sig={sig}" - assert "wait_for" in params, f"scrape() missing 'wait_for' param; sig={sig}" - - @staticmethod - def test_search_signature(): - """search() must accept query and limit.""" - import inspect - - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - sig = inspect.signature(client.search) - params = sig.parameters - - assert "query" in params, f"search() missing 'query' param; sig={sig}" - assert "limit" in params, f"search() missing 'limit' param; sig={sig}" - - @staticmethod - def test_extract_signature(): - """extract() must accept urls, prompt, and schema.""" - import inspect - - from firecrawl import Firecrawl - - client = Firecrawl(api_key="test-contract-key") - sig = inspect.signature(client.extract) - params = sig.parameters - - assert "urls" in params, f"extract() missing 'urls' param; sig={sig}" - assert "prompt" in params, f"extract() missing 'prompt' param; sig={sig}" - assert "schema" in params, f"extract() missing 'schema' param; sig={sig}"