from __future__ import annotations import asyncio import json import logging import socket import uuid from dataclasses import replace from datetime import UTC, datetime, timedelta from deerflow.constants import ( MCP_TASK_POLL_AFTER_MAX_SECONDS, MCP_TASK_REMOTE_ID_MAX_LENGTH, MCP_TASK_RESULT_ARTIFACT_MAX_BYTES, ) from deerflow.mcp.tasks import ( McpTaskDriverRegistry, McpTaskProtocolError, TaskReference, TaskSnapshot, TaskStatus, TaskSubmitRequest, ) from deerflow.persistence.mcp_tasks import DuplicateMcpRemoteTaskError logger = logging.getLogger(__name__) _MAX_PERSISTED_ERROR_CHARS = 4_000 _MAX_INPUT_REQUIRED_BYTES = 65_536 def _bound_error(error: str | None) -> str | None: if error is None: return None return error[:_MAX_PERSISTED_ERROR_CHARS] class McpTaskService: """Persist and poll long-running MCP tasks outside the Agent loop.""" def __init__( self, *, repository, drivers: McpTaskDriverRegistry, poll_interval_seconds: int, lease_seconds: int, max_concurrent_polls: int, max_poll_backoff_seconds: int = 300, input_required_poll_interval_seconds: int = 60, tracking_degraded_after_errors: int = 3, max_result_bytes: int = 65_536, result_preview_max_chars: int = 2_000, ) -> None: self._repository = repository self._drivers = drivers self._poll_interval_seconds = poll_interval_seconds self._lease_seconds = lease_seconds self._max_concurrent_polls = max_concurrent_polls self._max_poll_backoff_seconds = max_poll_backoff_seconds self._input_required_poll_interval_seconds = input_required_poll_interval_seconds self._tracking_degraded_after_errors = tracking_degraded_after_errors self._max_result_bytes = max_result_bytes self._result_preview_max_chars = result_preview_max_chars self._lease_owner = f"{socket.gethostname()}:{uuid.uuid4().hex}" self._task: asyncio.Task[None] | None = None self._stop = asyncio.Event() @property def drivers(self) -> McpTaskDriverRegistry: return self._drivers @property def tracking_degraded_after_errors(self) -> int: return self._tracking_degraded_after_errors async def submit( self, *, driver_name: str, request: TaskSubmitRequest, now: datetime | None = None, ) -> dict: """Submit through one driver and persist the remote handle before returning.""" driver = self._drivers.get(driver_name) if driver is None: raise LookupError(f"No MCP task driver registered as {driver_name!r}") submitted_at = now or datetime.now(UTC) local_task_id = request.local_task_id or f"mcp-task-{uuid.uuid4().hex}" driver_request = replace(request, local_task_id=local_task_id) submission = await driver.submit(driver_request) driver_data = {**request.driver_data, **submission.driver_data} task_reference = TaskReference( local_task_id=local_task_id, user_id=request.user_id, thread_id=request.thread_id, server_name=request.server_name, remote_task_id=submission.remote_task_id, driver_data=driver_data, ) try: if len(submission.remote_task_id) > MCP_TASK_REMOTE_ID_MAX_LENGTH: raise McpTaskProtocolError(f"MCP task remote_task_id must not exceed {MCP_TASK_REMOTE_ID_MAX_LENGTH} characters") snapshot = self._normalize_snapshot(submission.snapshot) next_poll_at = self._next_poll_at(snapshot, now=submitted_at) return await self._repository.create( task_id=local_task_id, user_id=request.user_id, thread_id=request.thread_id, run_id=request.run_id, tool_call_id=request.tool_call_id, server_name=request.server_name, driver_name=driver_name, remote_task_id=submission.remote_task_id, task_name=request.task_name, status=snapshot.status.value, result=snapshot.result, result_preview=snapshot.result_preview, result_truncated=snapshot.result_truncated, result_artifact=snapshot.result_artifact, error=snapshot.error, input_required=snapshot.input_required, next_poll_at=next_poll_at, driver_data=driver_data, ) except DuplicateMcpRemoteTaskError: # This handle already has a durable owner. Cancelling it as # compensation would terminate the pre-existing tracked task. raise except Exception: try: await driver.cancel(task_reference) except Exception: # noqa: BLE001 - preserve the original persistence failure logger.exception( "Failed to cancel untracked MCP task after persistence failure (task_id=%s, driver=%s, remote_task_id=%s)", local_task_id, driver_name, submission.remote_task_id, ) raise async def run_once(self, *, now: datetime) -> None: claimed = await self._repository.claim_due_tasks( now=now, lease_owner=self._lease_owner, lease_seconds=self._lease_seconds, limit=self._max_concurrent_polls, ) if not claimed: return results = await asyncio.gather( *(self._poll_one(task, now=now) for task in claimed), return_exceptions=True, ) for record, result in zip(claimed, results, strict=True): if isinstance(result, BaseException): logger.error( "Unexpected MCP task poll failure (task_id=%s); the lease will expire for recovery", record.get("id"), exc_info=(type(result), result, result.__traceback__), ) async def _poll_one(self, record: dict, *, now: datetime) -> None: driver_name = str(record.get("driver_name") or "") driver = self._drivers.get(driver_name) if driver is None: await self._release_after_error( record, now=now, error=f"No MCP task driver registered as {driver_name!r}", ) return try: snapshot = self._normalize_snapshot(await driver.get_status(TaskReference.from_record(record))) except McpTaskProtocolError as exc: logger.error( "MCP task status contract failed permanently (task_id=%s, driver=%s): %s", record.get("id"), driver_name, exc, ) await self._apply_snapshot( record, TaskSnapshot(status=TaskStatus.FAILED, error=_bound_error(str(exc))), polled_at=datetime.now(UTC), ) return except Exception as exc: # noqa: BLE001 - driver boundary; retry on the next poll polled_at = datetime.now(UTC) logger.warning( "MCP task status poll failed (task_id=%s, driver=%s); retrying", record.get("id"), driver_name, exc_info=True, ) await self._release_after_error(record, now=polled_at, error=str(exc) or type(exc).__name__) return polled_at = datetime.now(UTC) await self._apply_snapshot(record, snapshot, polled_at=polled_at) async def _apply_snapshot( self, record: dict, snapshot: TaskSnapshot, *, polled_at: datetime, ) -> None: applied = await self._repository.apply_snapshot( record["id"], lease_owner=self._lease_owner, status=snapshot.status.value, result=snapshot.result, result_preview=snapshot.result_preview, result_truncated=snapshot.result_truncated, result_artifact=snapshot.result_artifact, error=snapshot.error, input_required=snapshot.input_required, next_poll_at=self._next_poll_at(snapshot, now=polled_at), polled_at=polled_at, ) if not applied: logger.info( "Discarded MCP task poll result after lease ownership changed or expired (task_id=%s)", record.get("id"), ) def _next_poll_at(self, snapshot: TaskSnapshot, *, now: datetime) -> datetime | None: if not snapshot.is_pollable: return None interval = snapshot.poll_after_seconds or self._poll_interval_seconds if snapshot.status == TaskStatus.INPUT_REQUIRED: interval = max(interval, self._input_required_poll_interval_seconds) interval = min(interval, MCP_TASK_POLL_AFTER_MAX_SECONDS) return now + timedelta(seconds=interval) async def _release_after_error(self, record: dict, *, now: datetime, error: str) -> None: consecutive_errors = max(0, int(record.get("consecutive_poll_error_count") or 0)) retry_seconds = min( self._poll_interval_seconds * (2 ** min(consecutive_errors, 16)), self._max_poll_backoff_seconds, ) bounded_error = _bound_error(error) assert bounded_error is not None await self._repository.release_claim( record["id"], lease_owner=self._lease_owner, next_poll_at=now + timedelta(seconds=retry_seconds), error=bounded_error, ) def _normalize_snapshot(self, snapshot: TaskSnapshot) -> TaskSnapshot: """Bound remote payloads without ever storing truncated JSON.""" snapshot = replace(snapshot, error=_bound_error(snapshot.error)) if snapshot.result_artifact is not None: encoded_artifact = self._encode_json_payload( snapshot.result_artifact, field_name="result_artifact", ) if len(encoded_artifact) > MCP_TASK_RESULT_ARTIFACT_MAX_BYTES: raise McpTaskProtocolError(f"MCP task result_artifact payload exceeds the {MCP_TASK_RESULT_ARTIFACT_MAX_BYTES}-byte limit") if snapshot.input_required is not None: encoded_input = self._encode_json_payload( snapshot.input_required, field_name="input_required", ) if len(encoded_input) > _MAX_INPUT_REQUIRED_BYTES: raise McpTaskProtocolError(f"MCP task input_required payload exceeds the {_MAX_INPUT_REQUIRED_BYTES}-byte limit") if snapshot.result is None: return snapshot encoded = self._encode_json_payload(snapshot.result, field_name="result") if len(encoded) <= self._max_result_bytes: return snapshot if isinstance(snapshot.result, str): preview_source = snapshot.result else: preview_source = encoded.decode("utf-8", errors="replace") return replace( snapshot, result=None, result_preview=preview_source[: self._result_preview_max_chars], result_truncated=True, ) @staticmethod def _encode_json_payload(value, *, field_name: str) -> bytes: try: return json.dumps( value, ensure_ascii=False, separators=(",", ":"), allow_nan=False, ).encode("utf-8") except (TypeError, ValueError) as exc: raise McpTaskProtocolError(f"MCP task {field_name} is not valid JSON: {exc}") from exc async def start(self) -> None: if self._task is not None: return self._stop.clear() self._task = asyncio.create_task(self._run_loop(), name="deerflow-mcp-task-poller") async def stop(self) -> None: task = self._task if task is None: return self._stop.set() task.cancel() try: await task except asyncio.CancelledError: pass finally: self._task = None async def _run_loop(self) -> None: while not self._stop.is_set(): try: # The first pass runs immediately. Expired leases therefore # recover at startup without a separate destructive sweep. await self.run_once(now=datetime.now(UTC)) except Exception: logger.exception("MCP task poll failed; retrying next interval") try: await asyncio.wait_for( self._stop.wait(), timeout=self._poll_interval_seconds, ) except TimeoutError: continue