mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-04-25 19:28:23 +00:00
975b344ca7
* fix: resolve issue #651 - crawl error with None content handling Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system. The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by: 1. Validating HTTP responses from Jina API 2. Handling None/empty content at extraction stage 3. Adding fallback handling in Article markdown/message conversion 4. Improving error diagnostics with detailed logging 5. Adding 16 new tests with 100% coverage for critical paths * Update src/crawler/readability_extractor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/crawler/article.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
29 lines
865 B
Python
29 lines
865 B
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import logging
|
|
from readabilipy import simple_json_from_html_string
|
|
|
|
from .article import Article
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ReadabilityExtractor:
|
|
def extract_article(self, html: str) -> Article:
|
|
article = simple_json_from_html_string(html, use_readability=True)
|
|
|
|
content = article.get("content")
|
|
if not content or not str(content).strip():
|
|
logger.warning("Readability extraction returned empty content")
|
|
content = "<p>No content could be extracted from this page</p>"
|
|
|
|
title = article.get("title")
|
|
if not title or not str(title).strip():
|
|
title = "Untitled"
|
|
|
|
return Article(
|
|
title=title,
|
|
html_content=content,
|
|
)
|