mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
* feat(search): find conversations and jump to matching messages * test(extensions): align example registry with search contribution * test(search): scope bookmark search to its extension * fix(search): address review correctness and scaling gaps
161 lines
8.0 KiB
Python
161 lines
8.0 KiB
Python
"""Workspace search and direct navigation to an owner-visible message."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from dataclasses import asdict
|
|
from typing import Literal
|
|
|
|
from fastapi import APIRouter, HTTPException, Query, Request
|
|
from pydantic import BaseModel, Field
|
|
|
|
from app.gateway.authz import require_permission
|
|
from app.gateway.deps import get_run_event_store, get_run_manager, get_thread_store
|
|
from app.gateway.search_corpus import GatewaySearchCorpus
|
|
from deerflow.utils.thread_id import ThreadId
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api", tags=["search"])
|
|
|
|
|
|
class WorkspaceSearchRequest(BaseModel):
|
|
query: str = Field(default="", max_length=256)
|
|
types: list[Literal["thread", "message"]] = Field(default_factory=lambda: ["thread", "message"], min_length=1, max_length=2)
|
|
archived: bool | None = Field(default=None, strict=True)
|
|
project_id: str | None = Field(default=None, max_length=64)
|
|
limit: int = Field(default=20, ge=1, le=100)
|
|
cursor: str | None = Field(default=None, max_length=4096)
|
|
|
|
|
|
def snippet(text: str, query: str) -> list[dict]:
|
|
"""Plain text only. Map case-fold expansions back to source characters."""
|
|
normalized = text.casefold()
|
|
position = normalized.find(query.casefold()) if query else -1
|
|
if position < 0:
|
|
return [{"text": text[:200] + ("…" if len(text) > 200 else "")}]
|
|
offsets = []
|
|
for index, character in enumerate(text):
|
|
offsets.extend([index] * len(character.casefold()))
|
|
if len(offsets) >= position + len(query.casefold()):
|
|
break
|
|
start, end = offsets[position], offsets[position + len(query.casefold()) - 1] + 1
|
|
left, right = max(0, start - 65), min(len(text), end + 110)
|
|
return [{"text": ("…" if left else "") + text[left:start]}, {"text": text[start:end], "matched": True}, {"text": text[end:right] + ("…" if right < len(text) else "")}]
|
|
|
|
|
|
def _corpus(request: Request) -> GatewaySearchCorpus:
|
|
return GatewaySearchCorpus(get_thread_store(request), get_run_event_store(request), get_run_manager(request), str(request.state.auth.require_user().id), request=request)
|
|
|
|
|
|
def _encode_cursor(scope: str, phase: str, position) -> str:
|
|
return base64.urlsafe_b64encode(json.dumps([scope, phase, position]).encode()).decode()
|
|
|
|
|
|
@router.post("/search")
|
|
@require_permission("threads", "read")
|
|
async def search_workspace(body: WorkspaceSearchRequest, request: Request):
|
|
corpus = _corpus(request)
|
|
query = body.query.strip()
|
|
scope = hashlib.sha256(json.dumps([corpus.user_id, query, sorted(set(body.types)), body.archived, body.project_id, "project_id" in body.model_fields_set]).encode()).hexdigest()
|
|
phase, position = "thread", None
|
|
if body.cursor:
|
|
try:
|
|
saved_scope, phase, position = json.loads(base64.urlsafe_b64decode(body.cursor))
|
|
if saved_scope != scope or phase not in {"thread", "message"}:
|
|
raise ValueError
|
|
if phase == "thread" and (not isinstance(position, list) or len(position) != 3 or not isinstance(position[0], int) or not all(isinstance(v, str) for v in position[1:])):
|
|
raise ValueError
|
|
if phase == "message" and position is not None and not isinstance(position, str):
|
|
raise ValueError
|
|
except (ValueError, TypeError) as exc:
|
|
raise HTTPException(400, "Invalid search cursor") from exc
|
|
all_threads = await corpus.threads()
|
|
threads = tuple(t for t in all_threads if (body.archived is None or t.archived == body.archived) and ("project_id" not in body.model_fields_set or t.project_id == body.project_id))
|
|
lookup = {t.thread_id: t for t in threads}
|
|
providers = getattr(getattr(request.app.state, "extensions", None), "workspace_search_providers", ())
|
|
messages_available = bool(providers) and request.state.auth.has_permission("runs", "read")
|
|
response = {"items": [], "cursor": None, "indexing": False, "messages_available": messages_available}
|
|
|
|
def thread_hit(thread):
|
|
return {
|
|
"type": "thread",
|
|
"thread_id": thread.thread_id,
|
|
"title": thread.title,
|
|
"agent_name": thread.agent_name,
|
|
"archived": thread.archived,
|
|
"project_id": thread.project_id,
|
|
"updated_at": thread.updated_at,
|
|
"snippet": snippet(thread.title, query),
|
|
}
|
|
|
|
if phase == "thread" and "thread" in body.types:
|
|
|
|
def key(thread):
|
|
title = thread.title.casefold()
|
|
return (2 if query and title == query.casefold() else 1, thread.updated_at, thread.thread_id)
|
|
|
|
matches = sorted((t for t in threads if not query or query.casefold() in t.title.casefold()), key=key, reverse=True)
|
|
if position is not None:
|
|
matches = [t for t in matches if key(t) < tuple(position)]
|
|
selected = matches[: body.limit]
|
|
response["items"].extend(thread_hit(t) for t in selected)
|
|
if len(matches) > body.limit:
|
|
response["cursor"] = _encode_cursor(scope, "thread", key(selected[-1]))
|
|
return response
|
|
if not query or "message" not in body.types or not messages_available:
|
|
return response
|
|
if len(response["items"]) == body.limit:
|
|
response["cursor"] = _encode_cursor(scope, "message", None)
|
|
return response
|
|
provider = providers[0][1]
|
|
provider_cursor = position if phase == "message" else None
|
|
# Invalidated candidates must not consume a visible result slot. Keep a
|
|
# bounded amount of work per HTTP page and expose continuation when needed.
|
|
for _ in range(5):
|
|
try:
|
|
page = await provider.search(corpus, query=query, threads=threads, limit=body.limit - len(response["items"]), cursor=provider_cursor)
|
|
except ValueError as exc:
|
|
raise HTTPException(400, "Invalid search cursor") from exc
|
|
except Exception:
|
|
logger.exception("Workspace search provider failed")
|
|
raise HTTPException(503, "Message search is temporarily unavailable") from None
|
|
response["indexing"] = page.indexing
|
|
for candidate in page.items:
|
|
thread = lookup.get(candidate.thread_id)
|
|
if thread is None or candidate.incarnation != thread.incarnation or not await corpus.owns(thread):
|
|
continue
|
|
# Re-read authoritative visible text before returning ANY snippet.
|
|
current = await corpus.latest_message(thread, candidate.message_id)
|
|
if current is None or current.seq != candidate.seq or query.casefold() not in current.text.casefold():
|
|
continue
|
|
hit = thread_hit(thread)
|
|
hit.update(type="message", message_id=current.message_id, seq=current.seq, role=current.role, updated_at=current.created_at, snippet=snippet(current.text, query))
|
|
response["items"].append(hit)
|
|
if page.indexing:
|
|
break
|
|
if page.cursor == provider_cursor and page.cursor is not None:
|
|
raise HTTPException(503, "Search pagination did not advance")
|
|
provider_cursor = page.cursor
|
|
if provider_cursor is None or len(response["items"]) >= body.limit:
|
|
break
|
|
if provider_cursor:
|
|
response["cursor"] = _encode_cursor(scope, "message", provider_cursor)
|
|
return response
|
|
|
|
|
|
@router.get("/threads/{thread_id}/messages/around")
|
|
@require_permission("runs", "read", owner_check=True, require_existing=True)
|
|
async def message_context(thread_id: ThreadId, request: Request, message_id: str = Query(min_length=1, max_length=512), seq: int | None = Query(default=None, ge=1), limit: int = Query(default=20, ge=1, le=50)):
|
|
corpus = _corpus(request)
|
|
window = await corpus.around(thread_id, message_id, seq=seq, limit=limit)
|
|
if window is None:
|
|
raise HTTPException(404, "Message is unavailable")
|
|
thread = await corpus.thread(thread_id)
|
|
if thread is None:
|
|
raise HTTPException(404, "Message is unavailable")
|
|
messages = [asdict(message) for row in window.pop("data") if (message := corpus.project(thread, row)) is not None]
|
|
return {**window, "messages": messages}
|