mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 00:10:42 +00:00
* feat(mcp): bind agent credentials to approved definitions * fix(mcp): enforce live auth policy and pin task destinations --------- Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
268 lines
13 KiB
Python
268 lines
13 KiB
Python
"""Kill a real Gateway during a task call and recover through its HTTP API.
|
|
|
|
Only the model and remote OAuth/MCP service are deterministic fixtures. Auth,
|
|
admission, task submission, leases, SQLite recovery and both Gateway lifespans
|
|
run in production code, in distinct OS processes over loopback TCP.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import socket
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from contextlib import closing, contextmanager
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from urllib.parse import parse_qs
|
|
|
|
import httpx
|
|
import pytest
|
|
from support.mcp_agent_auth import agent_auth_env as agent_auth_env
|
|
|
|
from deerflow.config.agents_config import load_agent_config
|
|
from deerflow.mcp.agent_auth import agent_definition_digest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
pytestmark = pytest.mark.skipif(os.name != "posix", reason="Verifies SIGKILL and OS-process lease recovery")
|
|
|
|
|
|
def _until(check, *, seconds=30):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
result = check()
|
|
if result:
|
|
return result
|
|
# This polls observable HTTP state; correctness never depends on a delay.
|
|
time.sleep(0.05)
|
|
raise AssertionError(f"Timed out waiting for {check}")
|
|
|
|
|
|
@contextmanager
|
|
def _remote_service(operation):
|
|
state = SimpleNamespace(generation=1, reached=threading.Event(), release=threading.Event(), calls=[], tokens=[])
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_args):
|
|
pass
|
|
|
|
def reply(self, status, payload=None):
|
|
body = json.dumps(payload).encode() if payload is not None else b""
|
|
try:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
pass # The killed Gateway cannot receive the parked response.
|
|
|
|
def do_GET(self):
|
|
self.reply(405)
|
|
|
|
def do_POST(self):
|
|
body = self.rfile.read(int(self.headers["Content-Length"]))
|
|
if self.path == "/token":
|
|
form = parse_qs(body.decode())
|
|
if form.get("client_id") != ["research-client"] or form.get("client_secret") != ["test-client-secret"]:
|
|
return self.reply(401)
|
|
token = f"research-client-{len(state.tokens) + 1}"
|
|
state.tokens.append(token)
|
|
return self.reply(200, {"access_token": token, "expires_in": 3600})
|
|
message = json.loads(body)
|
|
if "id" not in message:
|
|
return self.reply(202)
|
|
method = message["method"]
|
|
if method == "initialize":
|
|
result = {"protocolVersion": message["params"]["protocolVersion"], "capabilities": {"tools": {}}, "serverInfo": {"name": "restart-test", "version": "1"}}
|
|
elif method == "tools/list":
|
|
result = {"tools": [{"name": name, "description": name, "inputSchema": {"type": "object", "properties": {} if name == "submit_job" else {"task_id": {"type": "string"}}}} for name in ("submit_job", "poll_job", "cancel_job")]}
|
|
elif method == "tools/call":
|
|
name = message["params"]["name"]
|
|
headers = self.headers.get_all("Authorization") or []
|
|
state.calls.append({"generation": state.generation, "tool": name, "authorization": headers})
|
|
if len(headers) != 1 or headers[0] not in ["Bearer " + token for token in state.tokens]:
|
|
return self.reply(401)
|
|
if state.generation == 1 and name == operation + "_job":
|
|
state.reached.set()
|
|
if not state.release.wait(timeout=45):
|
|
return self.reply(500, {"error": "Test barrier timed out"})
|
|
result = {"isError": False, "content": [], "structuredContent": {"task_id": "remote-1", "status": "cancelled" if name == "cancel_job" else "running", "poll_after_seconds": 1}}
|
|
else:
|
|
return self.reply(400)
|
|
self.reply(200, {"jsonrpc": "2.0", "id": message["id"], "result": result})
|
|
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield f"http://127.0.0.1:{server.server_port}", state
|
|
finally:
|
|
state.release.set()
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join(timeout=5)
|
|
|
|
|
|
@contextmanager
|
|
def _gateway(tmp_path, client, port, generation):
|
|
# Exclude ambient provider credentials and point dotenv/config/home at
|
|
# temporary files. The child resolves its model through normal config.
|
|
env = {key: os.environ[key] for key in ("PATH", "HOME", "TMPDIR", "LANG") if key in os.environ}
|
|
env.update(
|
|
PYTHONPATH=os.pathsep.join((str(ROOT / "backend"), str(ROOT / "backend/tests"))),
|
|
DEER_FLOW_HOME=str(tmp_path),
|
|
DEER_FLOW_PROJECT_ROOT=str(ROOT),
|
|
DEER_FLOW_CONFIG_PATH=str(tmp_path / "config.yaml"),
|
|
DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(tmp_path / "extensions.json"),
|
|
DEER_FLOW_ENV_FILE=str(tmp_path / "empty.env"),
|
|
DEER_FLOW_AUTH_DISABLED="0",
|
|
GATEWAY_WORKERS="1",
|
|
WEB_CONCURRENCY="1",
|
|
AUTH_JWT_SECRET="isolated-gateway-restart-test-secret-1234567890",
|
|
NO_PROXY="127.0.0.1,localhost", # Also overrides macOS system proxies.
|
|
PYTHONIOENCODING="utf-8",
|
|
)
|
|
log_path = tmp_path / f"gateway-{generation}.log"
|
|
with log_path.open("w", encoding="utf-8") as log:
|
|
process = subprocess.Popen([sys.executable, "-m", "uvicorn", "app.gateway.app:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=tmp_path, env=env, stdout=log, stderr=subprocess.STDOUT)
|
|
try:
|
|
|
|
def healthy():
|
|
assert process.poll() is None, log_path.read_text(encoding="utf-8")[-8000:]
|
|
try:
|
|
return client.get("/health").status_code == 200
|
|
except httpx.TransportError:
|
|
return False
|
|
|
|
_until(healthy, seconds=45)
|
|
yield process
|
|
finally:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait(timeout=10)
|
|
|
|
|
|
def _write_config(path, data):
|
|
replacement = path.with_suffix(".new")
|
|
replacement.write_text(json.dumps(data), encoding="utf-8")
|
|
replacement.replace(path)
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["poll", "cancel"])
|
|
@pytest.mark.parametrize("policy_change", ["definition", "revocation", "destination"])
|
|
def test_gateway_process_restart_keeps_task_identity_and_enforces_revocation(agent_auth_env, tmp_path, operation, policy_change):
|
|
revoked = policy_change == "revocation"
|
|
denied = policy_change != "definition"
|
|
env = agent_auth_env
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
with _remote_service(operation) as (remote_url, remote), httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=30, trust_env=False) as client:
|
|
data = env.config.model_dump(mode="json", exclude={"extensions"})
|
|
data.update(
|
|
models=[{"name": "offline", "use": "support.mcp_agent_auth:AgentAuthModel", "model": "offline", "supports_tool_calling": True}],
|
|
agents_api={"enabled": True},
|
|
database={"backend": "sqlite", "sqlite_dir": str(tmp_path / "db")},
|
|
title={"enabled": False},
|
|
memory={"enabled": False},
|
|
summarization={"enabled": False},
|
|
tool_search={"enabled": False},
|
|
mcp_tasks={"enabled": True, "poll_interval_seconds": 1, "lease_seconds": 5},
|
|
)
|
|
binding = data["mcp_agent_auth"]["research-tickets"]
|
|
binding["oauth"].update(token_url=remote_url + "/token", client_secret="test-client-secret")
|
|
config_path = tmp_path / "config.yaml"
|
|
_write_config(config_path, data)
|
|
server = env.config.extensions.mcp_servers["tickets"].model_dump(mode="json")
|
|
server.update(url=remote_url + "/mcp", task_toolsets=[{"name": "job", "submit_tool": "submit_job", "status_tool": "poll_job", "cancel_tool": "cancel_job"}])
|
|
(tmp_path / "extensions.json").write_text(json.dumps({"mcpServers": {"tickets": server}}), encoding="utf-8")
|
|
(tmp_path / "empty.env").touch()
|
|
|
|
def request(method, path, *, expected=200, **kwargs):
|
|
response = client.request(method, path, **kwargs)
|
|
assert response.status_code == expected, response.text
|
|
return response.json()
|
|
|
|
with _gateway(tmp_path, client, port, 1) as first:
|
|
owner = request("POST", "/api/v1/auth/register", expected=201, json={"email": "restart@example.com", "password": "isolated-test-password-123"})["id"]
|
|
client.headers["X-CSRF-Token"] = client.cookies["csrf_token"]
|
|
request("POST", "/api/agents", expected=201, json={"name": env.agent.name, "soul": env.soul, "tool_groups": ["mcp"], "skills": []})
|
|
agent = load_agent_config(env.agent.name, user_id=owner)
|
|
original_digest = agent_definition_digest(agent, env.soul)
|
|
binding.update(owners=[owner], definition_digests=[original_digest])
|
|
_write_config(config_path, data)
|
|
thread_id = request("POST", "/api/threads", json={})["thread_id"]
|
|
request(
|
|
"POST",
|
|
f"/api/threads/{thread_id}/runs/wait",
|
|
json={"assistant_id": "lead_agent", "input": {"messages": [{"role": "user", "content": "Submit a job."}]}, "context": {"agent_name": env.agent.name, "mode": "flash", "subagent_enabled": False}},
|
|
)
|
|
tasks = request("GET", f"/api/threads/{thread_id}/mcp-tasks")
|
|
assert len(tasks) == 1, tasks
|
|
task_url = f"/api/threads/{thread_id}/mcp-tasks/{tasks[0]['task_id']}"
|
|
if operation == "cancel":
|
|
request("POST", task_url + "/cancel")
|
|
assert remote.reached.wait(timeout=20), remote.calls
|
|
first.kill()
|
|
first.wait(timeout=10)
|
|
assert first.returncode == -signal.SIGKILL
|
|
|
|
# Inspect production writes only; never manufacture a recoverable row.
|
|
with closing(sqlite3.connect(tmp_path / "db/deerflow.db")) as connection:
|
|
driver_data = json.loads(connection.execute("SELECT driver_data FROM mcp_tasks WHERE id = ?", (tasks[0]["task_id"],)).fetchone()[0])
|
|
assert driver_data["agent_auth"]["definition_digest"] == original_digest
|
|
assert "test-client-secret" not in json.dumps(driver_data)
|
|
|
|
new_soul = "Reviewed definition revision two."
|
|
(tmp_path / "users" / owner / "agents" / env.agent.name / "SOUL.md").write_text(new_soul, encoding="utf-8")
|
|
binding["definition_digests"] = [agent_definition_digest(agent, new_soul)]
|
|
binding["enabled"] = not revoked
|
|
_write_config(config_path, data)
|
|
if policy_change == "destination":
|
|
server["url"] = remote_url + "/replacement-mcp"
|
|
_write_config(tmp_path / "extensions.json", {"mcpServers": {"tickets": server}})
|
|
remote.generation = 2
|
|
remote.release.set()
|
|
with _gateway(tmp_path, client, port, 2) as second:
|
|
assert second.pid != first.pid
|
|
target_status = "failed" if denied else "cancelled" if operation == "cancel" else "working"
|
|
|
|
def recovered():
|
|
record = request("GET", task_url)
|
|
sent = [call for call in remote.calls if call["generation"] == 2 and call["tool"] == operation + "_job"]
|
|
return record if record["status"] == target_status and (denied or sent) else None
|
|
|
|
result = _until(recovered)
|
|
if operation == "cancel":
|
|
assert result["cancel_attempt_count"] >= 2, result
|
|
later_calls = [call for call in remote.calls if call["generation"] == 2]
|
|
if denied:
|
|
assert result["error"], result
|
|
assert later_calls == []
|
|
assert remote.tokens == ["research-client-1"]
|
|
else:
|
|
assert any(call["tool"] == operation + "_job" for call in later_calls)
|
|
assert all(call["authorization"] == ["Bearer research-client-2"] for call in later_calls)
|
|
assert remote.tokens == ["research-client-1", "research-client-2"]
|
|
(tmp_path / "evidence.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"operation": operation,
|
|
"policy_change": policy_change,
|
|
"old_pid": first.pid,
|
|
"new_pid": second.pid,
|
|
"old_exit": first.returncode,
|
|
"task_status": result["status"],
|
|
"token_acquisitions": len(remote.tokens),
|
|
"calls": remote.calls,
|
|
},
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|