mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-27 13:40:37 +00:00
5841c8e9b7
* fix(channels): make Feishu and DingTalk transports stoppable in-process
FeishuChannel.stop() and DingTalkChannel.stop() could not end their SDK
threads. Both SDKs run on a thread-private event loop that only returned
on a fatal error: lark-oapi's ws.Client.start() parks forever in
run_until_complete(_select()) and has no public stop, and the channel
never kept the client handle; dingtalk-stream's start_forever() is a
while-True loop that also catches CancelledError, and the disconnect()
the channel called does not exist on the SDK. So every stop joined for
5s, raised ChannelStopTimeout, and ChannelService retained the instance
while the platform connection stayed alive on the old process. That
burns the full budget on shutdown and on POST /api/channels/{name}/restart,
and it blocks the per-provider leader lease: a demoted leader could not
release the transport before a peer took over.
Both channels now publish the SDK thread's loop and client under a lock;
stop() schedules a shutdown on that loop and then joins. Feishu disarms
lark's _auto_reconnect before closing the socket and stopping the loop,
because start() otherwise swallows the interrupted run_until_complete in
its reconnect branch and dials again. DingTalk drives its own copy of the
SDK connect loop that re-checks the running flag before every dial, runs
the blocking ticket request off the loop, cancels keepalive with the
socket, and logs the endpoint rather than the ticket-bearing payload. A
shared shutdown_private_loop cancels the SDK's leftover tasks with a
bounded drain and closes the loop; a stop requested before the client
exists never dials. ChannelStopTimeout now only fires for a thread that
genuinely wedges inside an SDK call. Telegram already stopped this way.
Tests: a fake lark ws.Client with the SDK's start() shape (including the
reconnect branch that swallows a bare loop stop), a real local websockets
server that observes the DingTalk close, the stop-before-client race for
both, and a blocking-io anchor that stop() ends a live private loop
without blocking the Gateway loop. All eight were red on main.
* docs(changelog): link the stoppable-transports entry to #6798
* fix(channels): release the socket on a stalled close and bound the DingTalk ticket request
Review follow-ups on #6798 (Totoro-qaq):
Feishu: the bounded _disconnect() wait cancelled websockets' close before
its own close-timeout abort ran, and lark clears _conn in its finally, so
a peer that never answers the close frame left the socket to garbage
collection. Worse, a stop landing while _connect() still held lark's
_lock snapshotted _conn as None. _shutdown_ws now settles the in-flight
dial by taking that lock under one shared close deadline, snapshots the
connection, runs _disconnect() under the same deadline, and aborts the
transport itself (abort_connection_transport in base.py) when the
handshake did not finish, yielding once so connection_lost runs before
loop.stop(). DingTalk's websocket.close() path gets the same abort.
DingTalk: the SDK's open_connection() calls requests.post with no
timeout, and running it in an executor let a stalled ticket endpoint
block interpreter exit (executor workers are joined at shutdown). The
channel now sends the same request through httpx on the stream loop
under a 30s deadline, so stop() cancels it and no worker thread lingers.
Tests drive the real SDK clients against loopback peers: a WebSocket
peer that completes the handshake and never ACKs the close (both
channels assert the connection's socket reports fileno() == -1 right
after stop(); red with the abort disabled), a real websockets server for
the clean path, and a loopback ticket server (ok / 500 / never responds)
with a no-lingering-non-daemon-thread check and a request-shape check.
1325 lines
58 KiB
Python
1325 lines
58 KiB
Python
"""DingTalk channel implementation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import platform
|
|
import re
|
|
import stat
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import quote_plus
|
|
|
|
import httpx
|
|
|
|
from app.channels.base import Channel, ChannelStopTimeout, abort_connection_transport, shutdown_private_loop
|
|
from app.channels.commands import is_known_channel_command, strip_leading_mentions
|
|
from app.channels.connection_identity import attach_connection_identity
|
|
from app.channels.message_bus import InboundMessage, InboundMessageType, InboundReservation, MessageBus, OutboundMessage, ResolvedAttachment
|
|
from app.channels.sandbox_files import sync_file_to_thread_sandbox
|
|
from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths
|
|
from deerflow.runtime.user_context import get_effective_user_id
|
|
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
|
|
from deerflow.uploads.manager import (
|
|
UnsafeUploadPathError,
|
|
apply_upload_sandbox_permits,
|
|
claim_unique_filename,
|
|
normalize_filename,
|
|
write_upload_file_no_symlink,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DINGTALK_API_BASE = "https://api.dingtalk.com"
|
|
|
|
_TOKEN_REFRESH_MARGIN_SECONDS = 300
|
|
|
|
# stop() budget for the dingtalk-stream thread: ask its private loop to close
|
|
# the socket (bounded by the close timeout) and stop, then join the thread.
|
|
DINGTALK_STOP_TIMEOUT_SECONDS = 5
|
|
DINGTALK_STREAM_CLOSE_TIMEOUT_SECONDS = 2
|
|
# Deadline for the Stream ticket request (the SDK's own has none).
|
|
DINGTALK_STREAM_TICKET_TIMEOUT_SECONDS = 30
|
|
# Retry pacing copied from DingTalkStreamClient.start() (dingtalk-stream 0.24).
|
|
_DINGTALK_STREAM_RETRY_SECONDS = 10
|
|
_DINGTALK_STREAM_ERROR_RETRY_SECONDS = 3
|
|
|
|
_CONVERSATION_TYPE_P2P = "1"
|
|
_CONVERSATION_TYPE_GROUP = "2"
|
|
|
|
_MAX_UPLOAD_SIZE_BYTES = 20 * 1024 * 1024
|
|
|
|
# Inbound attachments are buffered in memory before being persisted and synced
|
|
# into the sandbox, so bound them. DingTalk chats accept far larger files than
|
|
# an agent can usefully read; oversized ones surface as a failed-load marker.
|
|
_MAX_INBOUND_FILE_SIZE_BYTES = 50 * 1024 * 1024
|
|
|
|
|
|
def _normalize_conversation_type(raw: Any) -> str:
|
|
"""Normalize ``conversationType`` to ``"1"`` (P2P) or ``"2"`` (group).
|
|
|
|
Stream payloads may send int or string values.
|
|
"""
|
|
if raw is None:
|
|
return _CONVERSATION_TYPE_P2P
|
|
s = str(raw).strip()
|
|
if s == _CONVERSATION_TYPE_GROUP:
|
|
return _CONVERSATION_TYPE_GROUP
|
|
return _CONVERSATION_TYPE_P2P
|
|
|
|
|
|
def _normalize_allowed_users(allowed_users: Any) -> set[str]:
|
|
if allowed_users is None:
|
|
return set()
|
|
if isinstance(allowed_users, str):
|
|
values = [allowed_users]
|
|
elif isinstance(allowed_users, (list, tuple, set)):
|
|
values = allowed_users
|
|
else:
|
|
logger.warning(
|
|
"DingTalk allowed_users should be a list of user IDs; treating %s as one string value",
|
|
type(allowed_users).__name__,
|
|
)
|
|
values = [allowed_users]
|
|
return {str(uid) for uid in values if str(uid)}
|
|
|
|
|
|
def _is_dingtalk_command(text: str) -> bool:
|
|
return is_known_channel_command(text)
|
|
|
|
|
|
def _display_filename(filename: str) -> str:
|
|
"""Collapse whitespace and cap length for embedding a filename in message text.
|
|
|
|
``fileName`` is webhook-supplied data: embedded raw in a failed-load marker,
|
|
a newline could forge a standalone ``/mnt/user-data/uploads/...`` line and an
|
|
over-long name would bloat the message text.
|
|
"""
|
|
return re.sub(r"\s+", " ", filename).strip()[:80]
|
|
|
|
|
|
def _extract_text_from_rich_text(rich_text_list: list) -> str:
|
|
parts: list[str] = []
|
|
for item in rich_text_list:
|
|
if isinstance(item, dict) and "text" in item:
|
|
parts.append(item["text"])
|
|
return " ".join(parts)
|
|
|
|
|
|
_FENCED_CODE_BLOCK_RE = re.compile(r"```(\w*)\n(.*?)```", re.DOTALL)
|
|
_INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
|
|
_HORIZONTAL_RULE_RE = re.compile(r"^-{3,}$", re.MULTILINE)
|
|
_TABLE_SEPARATOR_RE = re.compile(r"^\|[-:| ]+\|$", re.MULTILINE)
|
|
|
|
|
|
def _split_markdown_table_row(row: str) -> list[str]:
|
|
r"""Keep escaped pipes and empty cells when removing the outer borders.
|
|
|
|
Known boundary: the single-character lookbehind in the split pattern
|
|
cannot distinguish an escaped pipe (``\|``) from an escaped backslash
|
|
immediately followed by a real separator (``\\|``). A cell ending in a
|
|
literal backslash therefore absorbs the next cell: ``| foo\\ | bar |``
|
|
splits as one cell instead of two. GFM counts consecutive backslashes
|
|
for this reason; doing so is out of scope here — this helper is
|
|
intentionally not a general Markdown parser.
|
|
"""
|
|
# ``(?<!\\)`` inspects only one preceding character, so the second
|
|
# backslash of ``\\|`` is mistaken for an escaping backslash and the
|
|
# pipe is kept whole. See the boundary note in the docstring above.
|
|
cells = re.split(r"(?<!\\)\|", row.strip())
|
|
if cells and cells[0] == "":
|
|
cells.pop(0)
|
|
if cells and cells[-1] == "":
|
|
cells.pop()
|
|
return [cell.strip() for cell in cells]
|
|
|
|
|
|
def _convert_markdown_table(text: str) -> str:
|
|
# DingTalk sampleMarkdown does not render pipe-delimited tables.
|
|
lines = text.split("\n")
|
|
result: list[str] = []
|
|
i = 0
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
# Detect table: header row followed by separator row
|
|
if i + 1 < len(lines) and line.strip().startswith("|") and _TABLE_SEPARATOR_RE.match(lines[i + 1].strip()):
|
|
headers = _split_markdown_table_row(line)
|
|
i += 2 # skip header + separator
|
|
while i < len(lines) and lines[i].strip().startswith("|"):
|
|
cells = _split_markdown_table_row(lines[i])
|
|
for h, c in zip(headers, cells):
|
|
result.append(f"> **{h}**: {c}")
|
|
result.append("")
|
|
i += 1
|
|
else:
|
|
result.append(line)
|
|
i += 1
|
|
return "\n".join(result)
|
|
|
|
|
|
def _adapt_markdown_for_dingtalk(text: str) -> str:
|
|
"""Adapt markdown for DingTalk's limited sampleMarkdown renderer."""
|
|
|
|
def _code_block_to_quote(match: re.Match) -> str:
|
|
lang = match.group(1)
|
|
code = match.group(2).rstrip("\n")
|
|
prefix = f"> **{lang}**\n" if lang else ""
|
|
quoted_lines = "\n".join(f"> {line}" for line in code.split("\n"))
|
|
return f"{prefix}{quoted_lines}\n"
|
|
|
|
text = _FENCED_CODE_BLOCK_RE.sub(_code_block_to_quote, text)
|
|
text = _INLINE_CODE_RE.sub(r"**\1**", text)
|
|
text = _convert_markdown_table(text)
|
|
text = _HORIZONTAL_RULE_RE.sub("───────────", text)
|
|
return text
|
|
|
|
|
|
class DingTalkChannel(Channel):
|
|
"""DingTalk IM channel using Stream Push (WebSocket, no public IP needed)."""
|
|
|
|
def __init__(self, bus: MessageBus, config: dict[str, Any]) -> None:
|
|
super().__init__(name="dingtalk", bus=bus, config=config)
|
|
self._thread: threading.Thread | None = None
|
|
self._main_loop: asyncio.AbstractEventLoop | None = None
|
|
self._client_id: str = ""
|
|
self._client_secret: str = ""
|
|
self._allowed_users: set[str] = _normalize_allowed_users(config.get("allowed_users"))
|
|
self._cached_token: str = ""
|
|
self._token_expires_at: float = 0.0
|
|
self._token_lock = asyncio.Lock()
|
|
self._card_template_id: str = config.get("card_template_id", "")
|
|
self._card_track_ids: dict[str, str] = {}
|
|
self._dingtalk_client: Any = None
|
|
# The SDK thread's private loop and client, published under _stream_lock
|
|
# once constructed so stop() can shut them down from the Gateway loop.
|
|
self._stream_client: Any = None
|
|
self._stream_loop: asyncio.AbstractEventLoop | None = None
|
|
self._stream_lock = threading.Lock()
|
|
self._incoming_messages: dict[str, Any] = {}
|
|
self._incoming_messages_lock = threading.Lock()
|
|
self._card_repliers: dict[str, Any] = {}
|
|
# Serialize inbound-file writes into the uploads directory to avoid
|
|
# racing writers clobbering one another (mirrors FeishuChannel).
|
|
self._file_write_lock = threading.Lock()
|
|
|
|
@property
|
|
def supports_streaming(self) -> bool:
|
|
return bool(self._card_template_id)
|
|
|
|
async def start(self) -> None:
|
|
if self._running:
|
|
return
|
|
|
|
try:
|
|
import dingtalk_stream # noqa: F401
|
|
except ImportError:
|
|
logger.error("dingtalk-stream is not installed. Install it with: uv add dingtalk-stream")
|
|
return
|
|
|
|
client_id = self.config.get("client_id", "")
|
|
client_secret = self.config.get("client_secret", "")
|
|
|
|
if not client_id or not client_secret:
|
|
logger.error("DingTalk channel requires client_id and client_secret")
|
|
return
|
|
|
|
self._client_id = client_id
|
|
self._client_secret = client_secret
|
|
self._main_loop = asyncio.get_running_loop()
|
|
|
|
if self._card_template_id:
|
|
logger.info("[DingTalk] AI Card mode enabled (template=%s)", self._card_template_id)
|
|
|
|
self._open_threadsafe_future_intake()
|
|
self._running = True
|
|
self.bus.subscribe_outbound(self._on_outbound)
|
|
|
|
self._thread = threading.Thread(
|
|
target=self._run_stream,
|
|
args=(client_id, client_secret),
|
|
daemon=True,
|
|
)
|
|
self._thread.start()
|
|
logger.info("DingTalk channel started")
|
|
|
|
async def stop(self) -> None:
|
|
self._running = False
|
|
self.bus.unsubscribe_outbound(self._on_outbound)
|
|
# Signal the SDK thread first so its loop winds down while the
|
|
# Gateway-side drain below runs.
|
|
self._request_stream_shutdown()
|
|
await self._close_and_drain_threadsafe_futures()
|
|
|
|
self._dingtalk_client = None
|
|
with self._incoming_messages_lock:
|
|
self._incoming_messages.clear()
|
|
self._card_repliers.clear()
|
|
self._card_track_ids.clear()
|
|
if self._thread:
|
|
# The thread exits once its loop has stopped; keep the join off the
|
|
# event loop because a wedged SDK call can still use up the budget.
|
|
thread = self._thread
|
|
await asyncio.to_thread(thread.join, timeout=DINGTALK_STOP_TIMEOUT_SECONDS)
|
|
if thread.is_alive():
|
|
raise ChannelStopTimeout("DingTalk SDK thread is still running after stop timeout")
|
|
if self._thread is thread:
|
|
self._thread = None
|
|
with self._stream_lock:
|
|
self._stream_loop = None
|
|
self._stream_client = None
|
|
logger.info("DingTalk channel stopped")
|
|
|
|
def _request_stream_shutdown(self) -> None:
|
|
"""Ask the SDK thread to close its socket and stop its loop (non-blocking)."""
|
|
with self._stream_lock:
|
|
loop = self._stream_loop
|
|
client = self._stream_client
|
|
if loop is None:
|
|
return
|
|
try:
|
|
loop.call_soon_threadsafe(self._begin_stream_shutdown, loop, client)
|
|
except RuntimeError:
|
|
# The loop closed between the snapshot and the call: the thread is
|
|
# already on its way out and the join below observes that.
|
|
logger.debug("[DingTalk] stream loop already closed before shutdown could be scheduled")
|
|
|
|
def _begin_stream_shutdown(self, loop: asyncio.AbstractEventLoop, client: Any) -> None:
|
|
loop.create_task(self._shutdown_stream(loop, client))
|
|
|
|
async def _shutdown_stream(self, loop: asyncio.AbstractEventLoop, client: Any) -> None:
|
|
"""Runs on the SDK loop: close the live websocket, then stop the loop."""
|
|
try:
|
|
websocket = getattr(client, "websocket", None) if client is not None else None
|
|
if websocket is not None:
|
|
try:
|
|
await asyncio.wait_for(websocket.close(), timeout=DINGTALK_STREAM_CLOSE_TIMEOUT_SECONDS)
|
|
except Exception:
|
|
logger.debug("[DingTalk] stream close did not complete cleanly", exc_info=True)
|
|
if abort_connection_transport(websocket):
|
|
logger.warning("[DingTalk] stream close handshake did not finish within %ss; transport aborted", DINGTALK_STREAM_CLOSE_TIMEOUT_SECONDS)
|
|
await asyncio.sleep(0) # let connection_lost run before the loop stops
|
|
except asyncio.CancelledError:
|
|
# The thread is already draining this loop; leave it to that.
|
|
raise
|
|
loop.stop()
|
|
|
|
def _resolve_routing(self, msg: OutboundMessage) -> tuple[str, str, str]:
|
|
"""Return (conversation_type, sender_staff_id, conversation_id).
|
|
|
|
Uses msg.chat_id as the primary routing key; metadata as fallback.
|
|
"""
|
|
conversation_type = _normalize_conversation_type(msg.metadata.get("conversation_type"))
|
|
sender_staff_id = msg.metadata.get("sender_staff_id", "")
|
|
conversation_id = msg.metadata.get("conversation_id", "")
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
conversation_id = msg.chat_id or conversation_id
|
|
else:
|
|
sender_staff_id = msg.chat_id or sender_staff_id
|
|
return conversation_type, sender_staff_id, conversation_id
|
|
|
|
async def send(self, msg: OutboundMessage, *, _max_retries: int = 3) -> None:
|
|
conversation_type, sender_staff_id, conversation_id = self._resolve_routing(msg)
|
|
robot_code = self._client_id
|
|
|
|
# Card mode: stream update to existing AI card
|
|
source_key = self._make_card_source_key_from_outbound(msg)
|
|
out_track_id = self._card_track_ids.get(source_key)
|
|
|
|
# ``card_template_id`` enables ``runs.stream`` (non-final + final outbounds).
|
|
# If card creation failed, skip non-final chunks to avoid duplicate messages.
|
|
if self._card_template_id and not out_track_id and not msg.is_final:
|
|
return
|
|
|
|
if out_track_id:
|
|
try:
|
|
await self._stream_update_card(
|
|
out_track_id,
|
|
msg.text,
|
|
is_finalize=msg.is_final,
|
|
)
|
|
except Exception:
|
|
logger.warning("[DingTalk] card stream failed, falling back to sampleMarkdown")
|
|
if msg.is_final:
|
|
self._card_track_ids.pop(source_key, None)
|
|
self._card_repliers.pop(out_track_id, None)
|
|
await self._send_markdown_fallback(robot_code, conversation_type, sender_staff_id, conversation_id, msg.text)
|
|
return
|
|
if msg.is_final:
|
|
self._card_track_ids.pop(source_key, None)
|
|
self._card_repliers.pop(out_track_id, None)
|
|
return
|
|
|
|
async def send_markdown() -> None:
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
await self._send_group_message(robot_code, conversation_id, msg.text, at_user_ids=[sender_staff_id] if sender_staff_id else None)
|
|
else:
|
|
await self._send_p2p_message(robot_code, sender_staff_id, msg.text)
|
|
|
|
# Non-card mode: send sampleMarkdown with retry
|
|
await self._send_with_retry(
|
|
send_markdown,
|
|
max_retries=_max_retries,
|
|
log_prefix="[DingTalk]",
|
|
)
|
|
return
|
|
|
|
async def _send_markdown_fallback(
|
|
self,
|
|
robot_code: str,
|
|
conversation_type: str,
|
|
sender_staff_id: str,
|
|
conversation_id: str,
|
|
text: str,
|
|
) -> None:
|
|
try:
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
await self._send_group_message(robot_code, conversation_id, text)
|
|
else:
|
|
await self._send_p2p_message(robot_code, sender_staff_id, text)
|
|
except Exception:
|
|
logger.exception("[DingTalk] markdown fallback also failed")
|
|
raise
|
|
|
|
async def send_file(self, msg: OutboundMessage, attachment: ResolvedAttachment) -> bool:
|
|
if attachment.size > _MAX_UPLOAD_SIZE_BYTES:
|
|
logger.warning("[DingTalk] file too large (%d bytes), skipping: %s", attachment.size, attachment.filename)
|
|
return False
|
|
|
|
conversation_type, sender_staff_id, conversation_id = self._resolve_routing(msg)
|
|
robot_code = self._client_id
|
|
|
|
try:
|
|
media_id = await self._upload_media(attachment.actual_path, "image" if attachment.is_image else "file")
|
|
if not media_id:
|
|
return False
|
|
|
|
if attachment.is_image:
|
|
msg_key = "sampleImageMsg"
|
|
msg_param = json.dumps({"photoURL": media_id})
|
|
else:
|
|
msg_key = "sampleFile"
|
|
msg_param = json.dumps(
|
|
{
|
|
"fileUrl": media_id,
|
|
"fileName": attachment.filename,
|
|
"fileSize": str(attachment.size),
|
|
}
|
|
)
|
|
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": msg_key,
|
|
"msgParam": msg_param,
|
|
"robotCode": robot_code,
|
|
"openConversationId": conversation_id,
|
|
},
|
|
)
|
|
else:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": msg_key,
|
|
"msgParam": msg_param,
|
|
"robotCode": robot_code,
|
|
"userIds": [sender_staff_id],
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
logger.info("[DingTalk] file sent: %s", attachment.filename)
|
|
return True
|
|
except (httpx.HTTPError, OSError, ValueError, TypeError, AttributeError):
|
|
logger.exception("[DingTalk] failed to send file: %s", attachment.filename)
|
|
return False
|
|
|
|
# -- stream client (runs in dedicated thread) --------------------------
|
|
|
|
def _run_stream(self, client_id: str, client_secret: str) -> None:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
try:
|
|
import dingtalk_stream
|
|
|
|
credential = dingtalk_stream.Credential(client_id, client_secret)
|
|
client = dingtalk_stream.DingTalkStreamClient(credential)
|
|
client.register_callback_handler(
|
|
dingtalk_stream.chatbot.ChatbotMessage.TOPIC,
|
|
_DingTalkMessageHandler(self),
|
|
)
|
|
with self._stream_lock:
|
|
if not self._running:
|
|
# stop() ran before the client existed: never dial.
|
|
return
|
|
self._stream_loop = loop
|
|
self._stream_client = client
|
|
loop.run_until_complete(self._serve_stream(client))
|
|
except Exception:
|
|
if self._running:
|
|
logger.exception("DingTalk Stream Push error")
|
|
else:
|
|
logger.debug("[DingTalk] stream loop exited after stop", exc_info=True)
|
|
self._running = False
|
|
finally:
|
|
with self._stream_lock:
|
|
if self._stream_loop is loop:
|
|
self._stream_loop = None
|
|
self._stream_client = None
|
|
shutdown_private_loop(loop)
|
|
|
|
async def _serve_stream(self, client: Any) -> None:
|
|
"""Run dingtalk-stream's connection loop so ``stop()`` can end it.
|
|
|
|
Mirrors ``DingTalkStreamClient.start()`` (dingtalk-stream 0.24): open a
|
|
connection ticket, hold the websocket, hand each frame to the SDK's
|
|
``background_task``, and retry after 10s on a lost connection or 3s on
|
|
anything else. The SDK's own loop is ``while True`` and also catches
|
|
``CancelledError``, so it can only be left by killing the thread; this
|
|
copy re-checks ``_running`` before every dial and fetches the ticket
|
|
asynchronously with a deadline so ``stop()`` can cancel it.
|
|
"""
|
|
import websockets
|
|
|
|
client.pre_start()
|
|
while self._running:
|
|
try:
|
|
connection = await self._open_stream_connection(client)
|
|
if not self._running:
|
|
break
|
|
if not connection:
|
|
logger.error("[DingTalk] open connection failed")
|
|
await asyncio.sleep(_DINGTALK_STREAM_RETRY_SECONDS)
|
|
continue
|
|
# The ticket in ``connection`` is a credential; log the endpoint only.
|
|
logger.info("[DingTalk] stream endpoint is %s", connection.get("endpoint"))
|
|
uri = f"{connection['endpoint']}?ticket={quote_plus(connection['ticket'])}"
|
|
async with websockets.connect(uri) as websocket:
|
|
client.websocket = websocket
|
|
keepalive = asyncio.create_task(client.keepalive(websocket))
|
|
try:
|
|
async for raw_message in websocket:
|
|
asyncio.create_task(client.background_task(json.loads(raw_message)))
|
|
finally:
|
|
keepalive.cancel()
|
|
except websockets.exceptions.ConnectionClosedError as exc:
|
|
if not self._running:
|
|
break
|
|
logger.error("[DingTalk] stream connection lost: %s", exc)
|
|
await asyncio.sleep(_DINGTALK_STREAM_RETRY_SECONDS)
|
|
except Exception:
|
|
if not self._running:
|
|
break
|
|
logger.exception("[DingTalk] stream error")
|
|
await asyncio.sleep(_DINGTALK_STREAM_ERROR_RETRY_SECONDS)
|
|
|
|
async def _open_stream_connection(self, client: Any) -> dict[str, Any] | None:
|
|
"""Fetch a Stream ticket: ``DingTalkStreamClient.open_connection()`` with a deadline.
|
|
|
|
The SDK's version calls ``requests.post`` with no timeout, so a stalled
|
|
ticket endpoint pins whichever thread runs it (and, from an executor,
|
|
Gateway process exit, which joins executor workers). This sends the same
|
|
request through ``httpx`` on the stream loop under
|
|
``DINGTALK_STREAM_TICKET_TIMEOUT_SECONDS``, so ``stop()`` cancels it like
|
|
any other task. Returns ``None`` on failure like the SDK does.
|
|
"""
|
|
from dingtalk_stream.version import VERSION_STRING
|
|
|
|
topics: list[dict[str, str]] = []
|
|
if getattr(client, "_is_event_required", False):
|
|
topics.append({"type": "EVENT", "topic": "*"})
|
|
for topic in client.callback_handler_map:
|
|
topics.append({"type": "CALLBACK", "topic": topic})
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json",
|
|
"User-Agent": f"DingTalkStream/1.0 SDK/{VERSION_STRING} Python/{platform.python_version()} (+https://github.com/open-dingtalk/dingtalk-stream-sdk-python)",
|
|
}
|
|
body = {
|
|
"clientId": client.credential.client_id,
|
|
"clientSecret": client.credential.client_secret,
|
|
"subscriptions": topics,
|
|
"ua": f"dingtalk-sdk-python/v{VERSION_STRING}-union",
|
|
"localIp": client.get_host_ip(),
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=DINGTALK_STREAM_TICKET_TIMEOUT_SECONDS) as http:
|
|
response = await http.post(client.OPEN_CONNECTION_API, headers=headers, json=body)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
except httpx.HTTPStatusError as exc:
|
|
logger.error("[DingTalk] open connection failed: HTTP %s", exc.response.status_code)
|
|
return None
|
|
except (httpx.HTTPError, ValueError) as exc:
|
|
logger.error("[DingTalk] open connection failed: %s", type(exc).__name__)
|
|
return None
|
|
if not isinstance(payload, dict) or not payload.get("endpoint") or not payload.get("ticket"):
|
|
logger.error("[DingTalk] open connection returned no endpoint/ticket")
|
|
return None
|
|
return payload
|
|
|
|
def _on_chatbot_message(self, message: Any) -> None:
|
|
if not self._running:
|
|
return
|
|
try:
|
|
sender_staff_id = message.sender_staff_id or ""
|
|
conversation_type = _normalize_conversation_type(message.conversation_type)
|
|
conversation_id = message.conversation_id or ""
|
|
msg_id = message.message_id or ""
|
|
sender_nick = message.sender_nick or ""
|
|
|
|
text = self._extract_text(message)
|
|
files = self._extract_files(message)
|
|
if not text and not files:
|
|
logger.info("[DingTalk] empty message with no files, ignoring")
|
|
return
|
|
|
|
connect_code = self._pending_connect_code(text)
|
|
if connect_code:
|
|
if self._main_loop and self._main_loop.is_running():
|
|
scheduled = self._submit_threadsafe_coroutine(
|
|
self._bind_connection_from_connect_code(
|
|
conversation_type=conversation_type,
|
|
sender_staff_id=sender_staff_id,
|
|
sender_nick=sender_nick,
|
|
conversation_id=conversation_id,
|
|
code=connect_code,
|
|
),
|
|
self._main_loop,
|
|
name="bind_connection",
|
|
msg_id=msg_id,
|
|
)
|
|
if not scheduled:
|
|
logger.info("[DingTalk] main loop stopped before channel connection bind could be scheduled")
|
|
else:
|
|
logger.warning("[DingTalk] main loop not running, cannot bind channel connection")
|
|
return
|
|
|
|
if self._allowed_users and sender_staff_id not in self._allowed_users:
|
|
logger.debug("[DingTalk] ignoring message from non-allowed user: %s", sender_staff_id)
|
|
return
|
|
|
|
# Log only metadata (length, not content) so message text never reaches
|
|
# INFO logs, and only after the allowed_users gate so blocked senders are
|
|
# not logged at all.
|
|
logger.info(
|
|
"[DingTalk] parsed message: conv_type=%s, msg_id=%s, sender=%s(%s), text_len=%d, files=%d",
|
|
conversation_type,
|
|
msg_id,
|
|
sender_staff_id,
|
|
sender_nick,
|
|
len(text or ""),
|
|
len(files),
|
|
)
|
|
|
|
# DingTalk group chats often deliver "@bot /new" with the mention left
|
|
# in the text (Slack/Discord strip their own bot mention upstream).
|
|
# Skip a leading mention only for the command path so ordinary chat
|
|
# keeps @mentions intact for the agent; the stripped form also flows
|
|
# into the inbound so ChannelManager._handle_command parses the bare
|
|
# command. Mirrors FeishuChannel.
|
|
command_text = strip_leading_mentions(text)
|
|
if _is_dingtalk_command(command_text):
|
|
msg_type = InboundMessageType.COMMAND
|
|
text = command_text
|
|
else:
|
|
msg_type = InboundMessageType.CHAT
|
|
|
|
# P2P: topic_id=None (single thread per user, like Telegram private chat)
|
|
# Group: topic_id=msg_id (each new message starts a new topic, like Feishu)
|
|
topic_id: str | None = msg_id if conversation_type == _CONVERSATION_TYPE_GROUP else None
|
|
|
|
# chat_id uses conversation_id for groups, sender_staff_id for P2P
|
|
chat_id = conversation_id if conversation_type == _CONVERSATION_TYPE_GROUP else sender_staff_id
|
|
|
|
# An empty chat_id does not identify a conversation, and ChannelStore keys on it:
|
|
# a P2P message with no sender keys to the literal "dingtalk:", so every such
|
|
# message from every user shares one thread and one history. Mirrors the guard the
|
|
# WeChat channel already applies (wechat.py, `if not chat_id: return`) and the one
|
|
# this file's own bind path applies before persisting a connection.
|
|
if not chat_id:
|
|
logger.warning(
|
|
"[DingTalk] ignoring message with no conversation identity: conv_type=%s, msg_id=%s",
|
|
conversation_type,
|
|
msg_id,
|
|
)
|
|
return
|
|
|
|
inbound = self._make_inbound(
|
|
chat_id=chat_id,
|
|
user_id=sender_staff_id,
|
|
text=text,
|
|
msg_type=msg_type,
|
|
thread_ts=msg_id,
|
|
files=files,
|
|
metadata={
|
|
"conversation_type": conversation_type,
|
|
"conversation_id": conversation_id,
|
|
"sender_staff_id": sender_staff_id,
|
|
"sender_nick": sender_nick,
|
|
"message_id": msg_id,
|
|
},
|
|
)
|
|
inbound.topic_id = topic_id
|
|
|
|
if self._main_loop and self._main_loop.is_running():
|
|
reservation = self._reserve_inbound(inbound)
|
|
if reservation is None:
|
|
return
|
|
if self._card_template_id:
|
|
source_key = self._make_card_source_key(inbound)
|
|
with self._incoming_messages_lock:
|
|
self._incoming_messages[source_key] = message
|
|
logger.info("[DingTalk] publishing inbound message to bus (type=%s, msg_id=%s)", msg_type.value, msg_id)
|
|
scheduled = self._submit_threadsafe_coroutine(
|
|
self._prepare_inbound(chat_id, inbound, reservation=reservation),
|
|
self._main_loop,
|
|
name="prepare_inbound",
|
|
msg_id=msg_id,
|
|
reservation=reservation,
|
|
)
|
|
if not scheduled:
|
|
logger.info("[DingTalk] main loop stopped before reserved inbound could be scheduled")
|
|
else:
|
|
logger.warning("[DingTalk] main loop not running, cannot publish inbound message")
|
|
except Exception:
|
|
logger.exception("[DingTalk] error processing chatbot message")
|
|
|
|
@staticmethod
|
|
def _extract_text(message: Any) -> str:
|
|
msg_type = message.message_type
|
|
if msg_type == "text" and message.text:
|
|
return message.text.content.strip()
|
|
if msg_type == "richText" and message.rich_text_content:
|
|
return _extract_text_from_rich_text(message.rich_text_content.rich_text_list).strip()
|
|
return ""
|
|
|
|
# -- inbound: file attachments -----------------------------------------
|
|
|
|
@staticmethod
|
|
def _extract_files(message: Any) -> list[dict[str, Any]]:
|
|
"""Extract inbound file/image descriptors from a DingTalk message.
|
|
|
|
Returns a list of dicts shaped for ``InboundMessage.files``; each carries
|
|
``type`` (``"image"`` or ``"file"``), ``download_code``, and ``filename``.
|
|
|
|
Images arrive as ``picture`` messages (a single ``downloadCode``) or as
|
|
inline images inside a ``richText`` message. Documents arrive as ``file``
|
|
messages, which ``dingtalk_stream.ChatbotMessage.from_dict`` does not
|
|
parse — the descriptor (``downloadCode`` / ``fileName``) is read from the
|
|
raw callback payload stashed on the message as ``_df_raw_data`` by
|
|
``_DingTalkMessageHandler.process``.
|
|
"""
|
|
msg_type = getattr(message, "message_type", None)
|
|
files: list[dict[str, Any]] = []
|
|
|
|
if msg_type == "picture":
|
|
image_content = getattr(message, "image_content", None)
|
|
code = getattr(image_content, "download_code", None)
|
|
if code:
|
|
files.append({"type": "image", "download_code": code, "filename": "image.png"})
|
|
elif msg_type == "richText":
|
|
try:
|
|
codes = message.get_image_list() or []
|
|
except Exception:
|
|
# Log rather than swallow: without this, a richText message whose
|
|
# SDK image parse fails looks identical to one that simply has no
|
|
# inline images, which is hard to diagnose if the SDK shape changes.
|
|
logger.warning("[DingTalk] failed to read inline images from richText message", exc_info=True)
|
|
codes = []
|
|
for idx, code in enumerate(codes):
|
|
if code:
|
|
files.append({"type": "image", "download_code": code, "filename": f"image_{idx}.png"})
|
|
elif msg_type == "file":
|
|
raw = getattr(message, "_df_raw_data", None)
|
|
content = raw.get("content") if isinstance(raw, dict) else None
|
|
if isinstance(content, dict):
|
|
code = content.get("downloadCode")
|
|
if code:
|
|
filename = content.get("fileName") or "file.bin"
|
|
files.append({"type": "file", "download_code": code, "filename": filename})
|
|
|
|
return files
|
|
|
|
async def receive_file(self, msg: InboundMessage, thread_id: str, *, user_id: str | None = None) -> InboundMessage:
|
|
"""Download inbound DingTalk files into the thread uploads directory.
|
|
|
|
Mirrors :meth:`FeishuChannel.receive_file`: each descriptor in
|
|
``msg.files`` is downloaded by its ``downloadCode``, persisted under the
|
|
thread's uploads bucket, synced into a non-local sandbox, and its sandbox
|
|
virtual path is prepended to ``msg.text`` so downstream models can read
|
|
the file by path. Descriptors are then cleared so the generic
|
|
URL-based ``_ingest_inbound_files`` pass does not try to re-fetch them
|
|
(DingTalk files are downloaded by code, not by URL).
|
|
|
|
An attachment that fails to download contributes a short
|
|
``[failed to load ...]`` marker instead of a path, so the agent can tell
|
|
the user something was sent but could not be read rather than silently
|
|
ignoring it.
|
|
"""
|
|
if not msg.files:
|
|
return msg
|
|
|
|
virtual_paths: list[str] = []
|
|
failures: list[str] = []
|
|
for f in msg.files:
|
|
if not isinstance(f, dict):
|
|
continue
|
|
download_code = f.get("download_code")
|
|
if not download_code:
|
|
continue
|
|
file_type = "image" if f.get("type") == "image" else "file"
|
|
filename = f.get("filename") if isinstance(f.get("filename"), str) else ""
|
|
# Per-attachment isolation: the manager awaits receive_file without a
|
|
# try, so anything escaping here would kill the chat turn with no
|
|
# reply — the silent-drop failure mode this feature exists to fix.
|
|
try:
|
|
virtual_path = await self._receive_single_file(download_code, file_type, filename, thread_id, user_id=user_id)
|
|
except Exception:
|
|
logger.exception("[DingTalk] unexpected error receiving inbound %s", file_type)
|
|
virtual_path = ""
|
|
if virtual_path:
|
|
virtual_paths.append(virtual_path)
|
|
else:
|
|
display = _display_filename(filename)
|
|
failures.append(f"[failed to load {file_type}: {display}]" if display else f"[failed to load {file_type}]")
|
|
|
|
prefix_lines = virtual_paths + failures
|
|
if prefix_lines:
|
|
block = "\n".join(prefix_lines)
|
|
msg.text = f"{block}\n\n{msg.text}".strip() if msg.text else block
|
|
|
|
msg.files = []
|
|
return msg
|
|
|
|
async def _receive_single_file(
|
|
self,
|
|
download_code: str,
|
|
file_type: str,
|
|
filename: str,
|
|
thread_id: str,
|
|
*,
|
|
user_id: str | None = None,
|
|
) -> str:
|
|
"""Download one file by ``downloadCode`` and persist it into uploads.
|
|
|
|
Returns the sandbox virtual path on success, or ``""`` on failure.
|
|
"""
|
|
content = await self._download_by_code(download_code)
|
|
if not content:
|
|
# Neutral on purpose: None covers several reasons (non-200
|
|
# download exchange, missing downloadUrl, oversize abort,
|
|
# transport failure), each already logged with its accurate
|
|
# reason inside _download_by_code — the empty-bytes case lands
|
|
# here too. Logging here keeps the skip observable at the
|
|
# receive level instead of vanishing silently (the wechat
|
|
# channel's callers and the manager reader use the same shape).
|
|
logger.warning("[DingTalk] inbound file download returned no content, skipping: file=%s", filename or "(unnamed)")
|
|
return ""
|
|
|
|
paths = get_paths()
|
|
effective_user_id = user_id or get_effective_user_id()
|
|
|
|
default_ext = "png" if file_type == "image" else "bin"
|
|
# ``download_code`` is attacker-controllable webhook data, so restrict it to
|
|
# a safe character set before embedding it in the fallback name. This keeps
|
|
# the fallback safe by construction instead of relying on a later write
|
|
# failure to reject a name that escaped the uploads directory.
|
|
code_token = re.sub(r"[^A-Za-z0-9_-]", "", download_code)[-12:] or "attachment"
|
|
fallback_name = f"dingtalk_{code_token}.{default_ext}"
|
|
# normalize_filename strips directory components and rejects traversal
|
|
# patterns ("..", backslash paths, over-long names); fall back to the
|
|
# sanitized generated name if the platform-supplied filename is rejected.
|
|
try:
|
|
safe_filename = normalize_filename(filename or fallback_name)
|
|
except ValueError:
|
|
safe_filename = fallback_name
|
|
|
|
def _persist() -> Path:
|
|
# Directory prep, the uniqueness claim, and the write are blocking
|
|
# filesystem IO — the whole sequence stays off the event loop.
|
|
# Exclusive creation also protects against publishers outside this
|
|
# channel instance's lock.
|
|
paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)
|
|
uploads_dir = paths.sandbox_uploads_dir(thread_id, user_id=effective_user_id).resolve()
|
|
with self._file_write_lock:
|
|
seen = {entry.name for entry in uploads_dir.iterdir()}
|
|
while True:
|
|
unique_name = claim_unique_filename(safe_filename, seen)
|
|
try:
|
|
return write_upload_file_no_symlink(uploads_dir, unique_name, content, exclusive=True)
|
|
except FileExistsError:
|
|
continue
|
|
|
|
try:
|
|
resolved_target = await asyncio.to_thread(_persist)
|
|
# Root-written uploads are 0o600, which the non-root sandbox cannot
|
|
# read on a bind-mounted thread dir; grant group/other read like the
|
|
# channel manager's inbound-file path and the HTTP upload route.
|
|
await asyncio.to_thread(apply_upload_sandbox_permits, resolved_target, stat.S_IRGRP | stat.S_IROTH)
|
|
except (OSError, UnsafeUploadPathError):
|
|
logger.exception("[DingTalk] failed to persist downloaded file: %s", safe_filename)
|
|
return ""
|
|
|
|
virtual_path = f"{VIRTUAL_PATH_PREFIX}/uploads/{resolved_target.name}"
|
|
|
|
try:
|
|
sandbox_provider = await asyncio.to_thread(get_sandbox_provider)
|
|
synced = await sync_file_to_thread_sandbox(
|
|
sandbox_provider,
|
|
thread_id=thread_id,
|
|
user_id=effective_user_id,
|
|
virtual_path=virtual_path,
|
|
content=content,
|
|
owner_prefix="dingtalk-upload",
|
|
)
|
|
if not synced:
|
|
# Mirror Feishu: the agent's non-local sandbox cannot see this
|
|
# file, so returning the virtual path would hand the model a
|
|
# path that reads as nothing — surface a failed-load marker.
|
|
logger.warning("[DingTalk] sandbox not found after acquire, dropping attachment: %s", virtual_path)
|
|
return ""
|
|
except Exception:
|
|
# Same failure mode as the sandbox-is-None branch: the bytes never
|
|
# reached the agent's sandbox, so the virtual path would read as
|
|
# nothing. Mirror Feishu and surface a failed-load marker.
|
|
logger.exception("[DingTalk] failed to sync downloaded file into non-local sandbox: %s", virtual_path)
|
|
return ""
|
|
|
|
return virtual_path
|
|
|
|
async def _download_by_code(self, download_code: str) -> bytes | None:
|
|
"""Exchange a DingTalk ``downloadCode`` for the raw file bytes.
|
|
|
|
Two steps per the DingTalk robot OpenAPI:
|
|
1. ``POST /v1.0/robot/messageFiles/download`` -> ``{downloadUrl}``
|
|
2. ``GET downloadUrl`` -> binary content
|
|
"""
|
|
try:
|
|
# Token acquisition stays inside the try: the manager awaits
|
|
# receive_file without a try, so a token failure escaping here would
|
|
# abort the whole chat turn instead of degrading to a failed-load marker.
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/messageFiles/download",
|
|
headers={
|
|
"x-acs-dingtalk-access-token": token,
|
|
"Content-Type": "application/json",
|
|
},
|
|
json={"downloadCode": download_code, "robotCode": self._client_id},
|
|
)
|
|
if response.status_code != 200:
|
|
logger.warning("[DingTalk] messageFiles/download failed: status=%d, body=%s", response.status_code, response.text[:300])
|
|
return None
|
|
download_url = response.json().get("downloadUrl")
|
|
if not download_url:
|
|
logger.warning("[DingTalk] messageFiles/download returned no downloadUrl")
|
|
return None
|
|
|
|
# Stream with a size cap: the bytes are buffered in memory before
|
|
# being persisted, so an oversized attachment must be refused
|
|
# before it is fully read, not after.
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
async with client.stream("GET", download_url, follow_redirects=True) as file_response:
|
|
file_response.raise_for_status()
|
|
async for chunk in file_response.aiter_bytes():
|
|
total += len(chunk)
|
|
if total > _MAX_INBOUND_FILE_SIZE_BYTES:
|
|
logger.warning("[DingTalk] inbound file exceeds %d bytes, dropping", _MAX_INBOUND_FILE_SIZE_BYTES)
|
|
return None
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
except (httpx.HTTPError, ValueError):
|
|
logger.exception("[DingTalk] failed to download file by code")
|
|
return None
|
|
|
|
async def _prepare_inbound(
|
|
self,
|
|
chat_id: str,
|
|
inbound: InboundMessage,
|
|
*,
|
|
reservation: InboundReservation | None = None,
|
|
) -> None:
|
|
try:
|
|
inbound = await self._attach_connection_identity(inbound)
|
|
# Running reply must finish before commit so AI card tracks are
|
|
# registered before the manager emits streaming outbounds.
|
|
await self._send_running_reply(chat_id, inbound)
|
|
if reservation is None:
|
|
await self.bus.publish_inbound(inbound)
|
|
else:
|
|
self._commit_reserved_inbound(reservation, inbound)
|
|
finally:
|
|
if reservation is not None:
|
|
reservation.release()
|
|
|
|
@staticmethod
|
|
def _connection_workspace_id(conversation_type: str, conversation_id: str) -> str | None:
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP and conversation_id:
|
|
return conversation_id
|
|
return None
|
|
|
|
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
|
conversation_type = str(inbound.metadata.get("conversation_type") or _CONVERSATION_TYPE_P2P)
|
|
conversation_id = str(inbound.metadata.get("conversation_id") or "")
|
|
return await attach_connection_identity(
|
|
inbound,
|
|
repo=self._connection_repo,
|
|
provider="dingtalk",
|
|
workspace_id=self._connection_workspace_id(conversation_type, conversation_id),
|
|
fallback_without_workspace=True,
|
|
)
|
|
|
|
async def _bind_connection_from_connect_code(
|
|
self,
|
|
*,
|
|
conversation_type: str,
|
|
sender_staff_id: str,
|
|
sender_nick: str,
|
|
conversation_id: str,
|
|
code: str,
|
|
) -> bool:
|
|
if self._connection_repo is None or not code:
|
|
return False
|
|
|
|
state = await self._connection_repo.consume_oauth_state(provider="dingtalk", state=code)
|
|
if state is None:
|
|
await self._send_connection_reply(
|
|
conversation_type,
|
|
sender_staff_id,
|
|
conversation_id,
|
|
"DingTalk connection code is invalid or expired.",
|
|
)
|
|
return True
|
|
|
|
if not sender_staff_id:
|
|
await self._send_connection_reply(
|
|
conversation_type,
|
|
sender_staff_id,
|
|
conversation_id,
|
|
"DingTalk connection could not be completed from this message.",
|
|
)
|
|
return True
|
|
|
|
await self._connection_repo.upsert_connection(
|
|
owner_user_id=state["owner_user_id"],
|
|
provider="dingtalk",
|
|
external_account_id=sender_staff_id,
|
|
external_account_name=sender_nick or None,
|
|
workspace_id=self._connection_workspace_id(conversation_type, conversation_id),
|
|
metadata={
|
|
"conversation_type": conversation_type,
|
|
"conversation_id": conversation_id,
|
|
},
|
|
status="connected",
|
|
)
|
|
await self._send_connection_reply(
|
|
conversation_type,
|
|
sender_staff_id,
|
|
conversation_id,
|
|
"DingTalk connected to DeerFlow.",
|
|
)
|
|
return True
|
|
|
|
async def _send_connection_reply(
|
|
self,
|
|
conversation_type: str,
|
|
sender_staff_id: str,
|
|
conversation_id: str,
|
|
text: str,
|
|
) -> None:
|
|
robot_code = self._client_id
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
if conversation_id:
|
|
await self._send_text_message_to_group(robot_code, conversation_id, text)
|
|
return
|
|
if sender_staff_id:
|
|
await self._send_text_message_to_user(robot_code, sender_staff_id, text)
|
|
|
|
async def _send_running_reply(self, chat_id: str, inbound: InboundMessage) -> None:
|
|
conversation_type = inbound.metadata.get("conversation_type", _CONVERSATION_TYPE_P2P)
|
|
sender_staff_id = inbound.metadata.get("sender_staff_id", "")
|
|
conversation_id = inbound.metadata.get("conversation_id", "")
|
|
text = "\u23f3 Working on it..."
|
|
|
|
try:
|
|
if self._card_template_id:
|
|
source_key = self._make_card_source_key(inbound)
|
|
with self._incoming_messages_lock:
|
|
chatbot_message = self._incoming_messages.pop(source_key, None)
|
|
out_track_id = await self._create_and_deliver_card(
|
|
text,
|
|
chatbot_message=chatbot_message,
|
|
)
|
|
if out_track_id:
|
|
self._card_track_ids[source_key] = out_track_id
|
|
logger.info("[DingTalk] AI card running reply sent for chat=%s", chat_id)
|
|
return
|
|
|
|
robot_code = self._client_id
|
|
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
|
await self._send_text_message_to_group(robot_code, conversation_id, text)
|
|
else:
|
|
await self._send_text_message_to_user(robot_code, sender_staff_id, text)
|
|
logger.info("[DingTalk] 'Working on it...' reply sent for chat=%s", chat_id)
|
|
except Exception:
|
|
logger.exception("[DingTalk] failed to send running reply for chat=%s", chat_id)
|
|
|
|
# -- DingTalk API helpers ----------------------------------------------
|
|
|
|
async def _get_access_token(self) -> str:
|
|
if self._cached_token and time.monotonic() < self._token_expires_at:
|
|
return self._cached_token
|
|
async with self._token_lock:
|
|
if self._cached_token and time.monotonic() < self._token_expires_at:
|
|
return self._cached_token
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/oauth2/accessToken",
|
|
json={"appKey": self._client_id, "appSecret": self._client_secret}, # DingTalk API field names
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
if not isinstance(data, dict):
|
|
raise ValueError(f"DingTalk access token response must be a JSON object, got {type(data).__name__}")
|
|
|
|
access_token = data.get("accessToken")
|
|
if not isinstance(access_token, str) or not access_token.strip():
|
|
raise ValueError("DingTalk access token response did not contain a usable accessToken")
|
|
|
|
raw_expires_in = data.get("expireIn", 7200)
|
|
try:
|
|
expires_in = int(raw_expires_in)
|
|
except (TypeError, ValueError):
|
|
logger.warning("[DingTalk] invalid expireIn value %r, using default 7200s", raw_expires_in)
|
|
expires_in = 7200
|
|
|
|
self._cached_token = access_token.strip()
|
|
self._token_expires_at = time.monotonic() + expires_in - _TOKEN_REFRESH_MARGIN_SECONDS
|
|
return self._cached_token
|
|
|
|
@staticmethod
|
|
def _api_headers(token: str) -> dict[str, str]:
|
|
return {
|
|
"x-acs-dingtalk-access-token": token,
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
async def _send_text_message_to_user(self, robot_code: str, user_id: str, text: str) -> None:
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": "sampleText",
|
|
"msgParam": json.dumps({"content": text}),
|
|
"robotCode": robot_code,
|
|
"userIds": [user_id],
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
async def _send_text_message_to_group(self, robot_code: str, conversation_id: str, text: str) -> None:
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": "sampleText",
|
|
"msgParam": json.dumps({"content": text}),
|
|
"robotCode": robot_code,
|
|
"openConversationId": conversation_id,
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
async def _send_p2p_message(self, robot_code: str, user_id: str, text: str) -> None:
|
|
text = _adapt_markdown_for_dingtalk(text)
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": "sampleMarkdown",
|
|
"msgParam": json.dumps({"title": "DeerFlow", "text": text}),
|
|
"robotCode": robot_code,
|
|
"userIds": [user_id],
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
if data.get("processQueryKey"):
|
|
logger.info("[DingTalk] P2P message sent to user=%s", user_id)
|
|
else:
|
|
logger.warning("[DingTalk] P2P send response: %s", data)
|
|
|
|
async def _send_group_message(
|
|
self,
|
|
robot_code: str,
|
|
conversation_id: str,
|
|
text: str,
|
|
*,
|
|
at_user_ids: list[str] | None = None, # noqa: ARG002
|
|
) -> None:
|
|
# at_user_ids accepted for call-site compatibility but not passed to the API
|
|
# (sampleMarkdown does not support @mentions).
|
|
text = _adapt_markdown_for_dingtalk(text)
|
|
token = await self._get_access_token()
|
|
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send",
|
|
headers=self._api_headers(token),
|
|
json={
|
|
"msgKey": "sampleMarkdown",
|
|
"msgParam": json.dumps({"title": "DeerFlow", "text": text}),
|
|
"robotCode": robot_code,
|
|
"openConversationId": conversation_id,
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
if data.get("processQueryKey"):
|
|
logger.info("[DingTalk] group message sent to conversation=%s", conversation_id)
|
|
else:
|
|
logger.warning("[DingTalk] group send response: %s", data)
|
|
|
|
# -- AI Card streaming helpers -------------------------------------------
|
|
|
|
def _make_card_source_key(self, inbound: InboundMessage) -> str:
|
|
m = inbound.metadata
|
|
return f"{m.get('conversation_type', '')}:{m.get('sender_staff_id', '')}:{m.get('conversation_id', '')}:{m.get('message_id', '')}"
|
|
|
|
def _make_card_source_key_from_outbound(self, msg: OutboundMessage) -> str:
|
|
m = msg.metadata
|
|
correlation_id = m.get("message_id") or msg.thread_ts or ""
|
|
return f"{m.get('conversation_type', '')}:{m.get('sender_staff_id', '')}:{m.get('conversation_id', '')}:{correlation_id}"
|
|
|
|
async def _create_and_deliver_card(
|
|
self,
|
|
initial_text: str,
|
|
*,
|
|
chatbot_message: Any = None,
|
|
) -> str | None:
|
|
if self._dingtalk_client is None or chatbot_message is None:
|
|
logger.warning("[DingTalk] SDK client or chatbot_message unavailable, skipping AI card")
|
|
return None
|
|
|
|
try:
|
|
from dingtalk_stream.card_replier import AICardReplier
|
|
except ImportError:
|
|
logger.warning("[DingTalk] dingtalk-stream card_replier not available")
|
|
return None
|
|
|
|
try:
|
|
replier = AICardReplier(self._dingtalk_client, chatbot_message)
|
|
card_instance_id = await replier.async_create_and_deliver_card(
|
|
card_template_id=self._card_template_id,
|
|
card_data={"content": initial_text},
|
|
)
|
|
if not card_instance_id:
|
|
return None
|
|
|
|
self._card_repliers[card_instance_id] = replier
|
|
logger.info("[DingTalk] AI card created: outTrackId=%s", card_instance_id)
|
|
return card_instance_id
|
|
except Exception:
|
|
logger.exception("[DingTalk] failed to create AI card")
|
|
return None
|
|
|
|
async def _stream_update_card(
|
|
self,
|
|
out_track_id: str,
|
|
content: str,
|
|
*,
|
|
is_finalize: bool = False,
|
|
is_error: bool = False,
|
|
) -> None:
|
|
replier = self._card_repliers.get(out_track_id)
|
|
if not replier:
|
|
raise RuntimeError(f"No AICardReplier found for track ID {out_track_id}")
|
|
|
|
await replier.async_streaming(
|
|
card_instance_id=out_track_id,
|
|
content_key="content",
|
|
content_value=content,
|
|
append=False,
|
|
finished=is_finalize,
|
|
failed=is_error,
|
|
)
|
|
|
|
# -- media upload --------------------------------------------------------
|
|
|
|
async def _upload_media(self, file_path: str | Path, media_type: str) -> str | None:
|
|
try:
|
|
file_bytes = await asyncio.to_thread(Path(file_path).read_bytes)
|
|
token = await self._get_access_token()
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(60.0)) as client:
|
|
response = await client.post(
|
|
f"{DINGTALK_API_BASE}/v1.0/files/upload",
|
|
headers={"x-acs-dingtalk-access-token": token},
|
|
files={"file": ("upload", file_bytes)},
|
|
data={"type": media_type},
|
|
)
|
|
response.raise_for_status()
|
|
try:
|
|
payload = response.json()
|
|
except json.JSONDecodeError:
|
|
logger.exception("[DingTalk] failed to decode upload response JSON: %s", file_path)
|
|
return None
|
|
if not isinstance(payload, dict):
|
|
logger.warning("[DingTalk] unexpected upload response type %s for %s", type(payload).__name__, file_path)
|
|
return None
|
|
return payload.get("mediaId")
|
|
except (httpx.HTTPError, OSError):
|
|
logger.exception("[DingTalk] failed to upload media: %s", file_path)
|
|
return None
|
|
|
|
|
|
class _DingTalkMessageHandler:
|
|
"""Callback handler registered with dingtalk-stream."""
|
|
|
|
def __init__(self, channel: DingTalkChannel) -> None:
|
|
self._channel = channel
|
|
|
|
def pre_start(self) -> None:
|
|
if hasattr(self, "dingtalk_client") and self.dingtalk_client is not None:
|
|
self._channel._dingtalk_client = self.dingtalk_client
|
|
|
|
async def raw_process(self, callback_message: Any) -> Any:
|
|
import dingtalk_stream
|
|
from dingtalk_stream.frames import Headers
|
|
|
|
code, message = await self.process(callback_message)
|
|
ack_message = dingtalk_stream.AckMessage()
|
|
ack_message.code = code
|
|
ack_message.headers.message_id = callback_message.headers.message_id
|
|
ack_message.headers.content_type = Headers.CONTENT_TYPE_APPLICATION_JSON
|
|
ack_message.data = {"response": message}
|
|
return ack_message
|
|
|
|
async def process(self, callback: Any) -> tuple[int, str]:
|
|
import dingtalk_stream
|
|
|
|
incoming_message = dingtalk_stream.ChatbotMessage.from_dict(callback.data)
|
|
# Stash the raw callback payload: dingtalk_stream does not parse ``file``
|
|
# (document) messages, so DingTalkChannel._extract_files reads the file
|
|
# descriptor (downloadCode / fileName) from it.
|
|
incoming_message._df_raw_data = callback.data
|
|
self._channel._on_chatbot_message(incoming_message)
|
|
return dingtalk_stream.AckMessage.STATUS_OK, "OK"
|