mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
f39cdd6077
* fix(mcp): keep rotating OAuth refresh tokens across restarts and Gateway instances
MCP servers on the OAuth refresh_token grant lost their rotated refresh
token whenever a new process or manager started (restart, tool-cache
reset for non-task servers, every personal-connection run, a second
Gateway instance) and re-seeded with the consumed configured token,
getting invalid_grant for good.
- New table mcp_oauth_tokens (migration 0047_mcp_oauth_tokens), one row
per refresh_token connection keyed (scope, owner_user_id, server_key):
('deployment', '', name) or ('personal', owner, personal runtime
name). Access and refresh tokens are DEER_FLOW_CREDENTIALS_KEY
ciphertext; a config fingerprint (token_url, client_id, grant, scope,
audience, seed) makes a changed seed win.
- OAuthTokenManager re-reads the row under its process lock, adopts a
peer's valid token, and otherwise claims the refresh with a lease plus
a version compare-and-set (database clock), fetches outside any
transaction, writes under the lease and releases it on failure.
- The Gateway installs the store (sync engine via to_thread) around the
LangGraph runtime for sqlite/postgres; personal managers pass their
owner. client_credentials stays in process; an unloadable key falls
back to process memory; database errors fail the refresh.
- credentials_key_consumers lists refresh_token MCP OAuth servers, so a
declared multi-instance deployment needs the shared key.
Docs: MCP_SERVER.md, CONFIGURATION.md, Helm README, README, mcp and
migrations AGENTS.md, CHANGELOG (en/zh).
* docs(changelog): link the MCP OAuth token store entry to #7207
* fix(mcp): prune removed connections' OAuth token rows and ask owners to re-authorize on invalid_grant
Review follow-up (willem-bd):
- Both MCP config writers drop mcp_oauth_tokens rows of connections a
save removes (deployment: _commit_mcp_config_write; personal:
_mutate, keeping each connection's enabled/disabled runtime-name
variants so a toggle keeps its chain). Best effort; no startup sweep,
since a missing config file would look like every server removed.
- invalid_grant on a refresh_token request raises
McpOAuthReauthorizationRequired: the tool error names the server and
asks for a new refresh token, plus the shared-key hint when this
instance could not decrypt the stored chain.
- Annotate the gateway helpers' config as AppConfig.
* fix(mcp): renew the OAuth refresh lease, keep unstored rotations, and key access tokens by request parameters
Review follow-up (willem-bd, second round):
- HTTPX timeouts bound each I/O wait, not a whole response: the refresh
holder now renews its lease every third of REFRESH_LEASE_SECONDS while
the token request runs and abandons it after REFRESH_MAX_SECONDS (120 s);
waiters outlast a live holder. Regression uses a real loopback endpoint
that streams its body for three lease lengths.
- A refresh whose tokens could not be stored keeps them pending
(unsaved_base_version): they are written back before an older stored
token is adopted or refreshed with, retried every few seconds while the
access token is still valid, and dropped once a peer stored a newer
chain. A failed save releases the lease.
- New token_request_fingerprint (extra_token_params and response fields):
a stored access token is reused only under the same request parameters;
otherwise the stored refresh-token chain is refreshed.
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
58 lines
3.5 KiB
Python
58 lines
3.5 KiB
Python
"""Additive Skill host schema; the chain-head pin lives in ``test_migration_0047_mcp_oauth_tokens``."""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
import sqlalchemy as sa
|
|
from alembic import command
|
|
from alembic.script import ScriptDirectory
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
from deerflow.persistence import bootstrap
|
|
from deerflow.persistence.run.model import RunRow
|
|
|
|
REVISION = "0046_skill_mutations"
|
|
TABLES = {"skill_mutation_assets", "skill_mutation_owners", "skill_mutation_proposals", "skill_mutation_operations", "skill_mutation_scan_attempts"}
|
|
|
|
|
|
def test_0046_remains_in_the_single_migration_chain():
|
|
script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR))
|
|
assert len(script.get_heads()) == 1
|
|
assert script.get_revision("0047_mcp_oauth_tokens").down_revision == REVISION
|
|
assert script.get_revision(REVISION).down_revision == "0045_completed_run_evidence"
|
|
assert script.get_revision("0045_completed_run_evidence").down_revision == "0044_channel_runtime_configs"
|
|
assert script.get_revision("0044_channel_runtime_configs").down_revision == "0043_runtime_im_notices"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_0046_additive_upgrade_and_nullable_recovery_metadata(tmp_path):
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'skill-host.db'}")
|
|
config = bootstrap._get_alembic_config(engine)
|
|
try:
|
|
await asyncio.to_thread(bootstrap._upgrade, config, "0045_completed_run_evidence")
|
|
# The historical create_all bootstrap uses today's metadata. Remove
|
|
# exactly the new, empty tables to represent a real 0045 installation.
|
|
async with engine.begin() as connection:
|
|
for table in sorted(TABLES):
|
|
await connection.execute(sa.text(f'DROP TABLE IF EXISTS "{table}"'))
|
|
await connection.execute(RunRow.__table__.insert().values(run_id="legacy", thread_id="thread", status="success"))
|
|
await connection.execute(sa.text("INSERT INTO channel_runtime_configs VALUES ('telegram', 'synthetic-ciphertext', 0, 7, 1.0)"))
|
|
await asyncio.to_thread(bootstrap._upgrade, config, REVISION)
|
|
async with engine.connect() as connection:
|
|
names = await connection.run_sync(lambda sync: sa.inspect(sync).get_table_names())
|
|
assert TABLES <= set(names)
|
|
columns = await connection.run_sync(lambda sync: sa.inspect(sync).get_columns("skill_mutation_operations"))
|
|
nullable = {column["name"]: column["nullable"] for column in columns}
|
|
assert nullable["before_operation_id"]
|
|
assert nullable["superseded_by_generation"]
|
|
assert await connection.scalar(sa.text("SELECT status FROM runs WHERE run_id='legacy'")) == "success"
|
|
assert (await connection.execute(sa.text("SELECT config_ciphertext, version FROM channel_runtime_configs WHERE provider='telegram'"))).one() == ("synthetic-ciphertext", 7)
|
|
await asyncio.to_thread(command.downgrade, config, "0045_completed_run_evidence")
|
|
async with engine.connect() as connection:
|
|
names = await connection.run_sync(lambda sync: sa.inspect(sync).get_table_names())
|
|
assert not TABLES.intersection(names)
|
|
assert await connection.scalar(sa.text("SELECT status FROM runs WHERE run_id='legacy'")) == "success"
|
|
assert (await connection.execute(sa.text("SELECT config_ciphertext, version FROM channel_runtime_configs WHERE provider='telegram'"))).one() == ("synthetic-ciphertext", 7)
|
|
finally:
|
|
await engine.dispose()
|