deer-flow/backend/app/gateway/search_corpus.py
Wenchao An dba950443d
feat(search): find conversations and jump to matching messages (#8032)
* feat(search): find conversations and jump to matching messages

* test(extensions): align example registry with search contribution

* test(search): scope bookmark search to its extension

* fix(search): address review correctness and scaling gaps
2026-11-13 23:01:38 +08:00

173 lines
8.3 KiB
Python

"""Permission-scoped transcript projection shared by search and message navigation."""
from __future__ import annotations
import hashlib
import json
import re
from deerflow_extension_api.search import SearchMessage, SearchMessagePage, SearchThread
from app.gateway.conversation_access import _visible_text
from app.gateway.conversation_reader import default_history_hidden_run_ids, read_visible_message_page
from deerflow.runtime.events.message_identity import message_identity
from deerflow.utils.time import coerce_iso
def visible_search_text(row: dict, *, thread_id: str):
"""Use the transcript reader and the chat UI's display-only exclusions."""
content = row.get("content")
if isinstance(content, dict) and content.get("name") in {"summary", "loop_warning", "todo_reminder", "todo_completion_reminder"}:
return None
visible = _visible_text(row, thread_id=thread_id)
if visible and visible[0] == "user":
text = re.sub(r"<(current_uploads|uploaded_files|slash_skill_activation)>[\s\S]*?</\1>", "", visible[1]).strip()
return ("user", text) if text else None
return visible
class GatewaySearchCorpus:
def __init__(self, thread_store, event_store, run_manager, user_id: str, *, request=None):
self.thread_store = thread_store
self.event_store = event_store
self.run_manager = run_manager
self.user_id = user_id
self.request = request
async def threads(self) -> tuple[SearchThread, ...]:
result = []
offset = 0
while True:
rows = await self.thread_store.search(user_id=self.user_id, limit=200, offset=offset)
for row in rows:
metadata = row.get("metadata") or {}
if metadata.get("deerflow_sidecar") is True:
continue
result.append(self.project_thread(row))
offset += len(rows)
if len(rows) < 200:
return tuple(result)
@staticmethod
def project_thread(row) -> SearchThread:
from app.gateway.services import custom_agent_name
try:
agent_name = custom_agent_name(row.get("assistant_id"))
except (TypeError, ValueError):
agent_name = None
metadata = row.get("metadata") or {}
return SearchThread(
thread_id=row["thread_id"],
incarnation=str(row.get("incarnation") or row.get("created_at") or ""),
title=row.get("display_name") or "",
updated_at=coerce_iso(row.get("updated_at", "")),
agent_name=agent_name,
project_id=metadata.get("deerflow_project_id"),
archived=metadata.get("deerflow_archived") is True,
)
async def thread(self, thread_id: str) -> SearchThread | None:
row = await self.thread_store.get(thread_id, user_id=self.user_id)
if row is None or (row.get("metadata") or {}).get("deerflow_sidecar") is True:
return None
return self.project_thread(row)
async def seed_legacy_history(self, thread: SearchThread) -> None:
if self.request is None or await self.event_store.list_messages(thread.thread_id, limit=1, user_id=None):
return
row = await self.thread_store.get(thread.thread_id, user_id=self.user_id)
if row is None or self.project_thread(row).incarnation != thread.incarnation:
return
from app.gateway.services import ensure_checkpoint_history_seeded
await ensure_checkpoint_history_seeded(self.request, thread_id=thread.thread_id, assistant_id=row.get("assistant_id"))
async def owns(self, thread: SearchThread) -> bool:
row = await self.thread_store.get(thread.thread_id, user_id=self.user_id)
return bool(row and str(row.get("incarnation") or row.get("created_at") or "") == thread.incarnation and (row.get("metadata") or {}).get("deerflow_sidecar") is not True)
async def revision(self, thread: SearchThread) -> str | None:
if not await self.owns(thread):
return None
await self.seed_legacy_history(thread)
latest = await self.event_store.list_messages(thread.thread_id, limit=1, user_id=self.user_id)
hidden = await default_history_hidden_run_ids(self.run_manager, thread.thread_id, user_id=self.user_id)
revision = [thread.incarnation, thread.updated_at, latest[-1]["seq"] if latest else 0, sorted(hidden)]
return hashlib.sha256(json.dumps(revision).encode()).hexdigest()
@staticmethod
def project(thread: SearchThread, row: dict) -> SearchMessage | None:
visible = visible_search_text(row, thread_id=thread.thread_id)
content = row.get("content")
identity = message_identity(content) if isinstance(content, dict) else None
if not visible or not identity:
return None
return SearchMessage(thread.thread_id, thread.incarnation, identity.removeprefix("message:"), row["run_id"], row["seq"], visible[0], visible[1], coerce_iso(row.get("created_at", "")))
async def messages(self, thread: SearchThread, *, before_seq: int | None) -> SearchMessagePage:
if not await self.owns(thread):
return SearchMessagePage()
rows, more = await self._page(thread.thread_id, limit=200, before_seq=before_seq)
items = tuple(message for row in rows if (message := self.project(thread, row)) is not None)
return SearchMessagePage(items, rows[0]["seq"] if more else None)
async def _page(self, thread_id: str, *, limit: int, before_seq=None, after_seq=None):
return await read_visible_message_page(
event_store=self.event_store,
run_manager=self.run_manager,
thread_id=thread_id,
user_id=self.user_id,
limit=limit,
before_seq=before_seq,
after_seq=after_seq,
message_filter=lambda row: visible_search_text(row, thread_id=thread_id) is not None and message_identity(row.get("content", {})) is not None,
)
async def around(self, thread_id: str, message_id: str, *, limit: int = 20, seq: int | None = None) -> dict | None:
row = await self.thread_store.get(thread_id, user_id=self.user_id)
if row is None or (row.get("metadata") or {}).get("deerflow_sidecar") is True:
return None
await self.seed_legacy_history(self.project_thread(row))
# Walk backward from the head so later versions win even when they
# live on another storage page. Sequence hints never select old text.
rows = await self.latest_rows(thread_id)
index = next((i for i, item in enumerate(rows) if message_identity(item["content"]) == f"message:{message_id}"), None)
if index is None or (seq is not None and rows[index]["seq"] != seq):
return None
target = rows[index]
seq = target["seq"]
left, right = max(0, index - limit), min(len(rows), index + limit + 1)
older, newer = rows[left:index], rows[index + 1 : right]
has_before, has_after = left > 0, right < len(rows)
# Recheck ownership/incarnation after the asynchronous history reads.
current = await self.thread_store.get(thread_id, user_id=self.user_id)
if current is None or current.get("incarnation") != row.get("incarnation"):
return None
return {"data": older + [target] + newer, "target_seq": seq, "has_before": has_before, "has_after": has_after}
async def latest_rows(self, thread_id: str) -> list[dict]:
latest = {}
before = None
while True:
rows, more = await self._page(thread_id, limit=200, before_seq=before)
for row in reversed(rows):
latest.setdefault(message_identity(row["content"]), row)
if not more:
break
before = rows[0]["seq"]
return sorted(latest.values(), key=lambda row: row["seq"])
async def latest_message(self, thread: SearchThread, message_id: str) -> SearchMessage | None:
if not await self.owns(thread):
return None
before = None
while True:
rows, more = await self._page(thread.thread_id, limit=200, before_seq=before)
for row in reversed(rows):
if message_identity(row["content"]) == f"message:{message_id}":
return self.project(thread, row)
if not more:
return None
before = rows[0]["seq"]