mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-06-09 17:12:01 +00:00
88759015e4
* test(e2e): record/replay front-back contract verification Guards the front-back contract with a deterministic, key-free record/replay harness (mirrors open-design's golden-trace approach): - ReplayChatModel (tests/replay_provider.py): replays recorded LLM turns by a normalized hash of the model input. Strips <system-reminder>/date/uuid/tmp-path so one fixture replays across days and from both the browser and direct-POST paths; a miss raises loudly (no silent divergence). - Recording is record-through-browser (scripts/record_gateway.py + build_fixture_from_jsonl.py + frontend/tests/e2e-record): a real run is driven through the real frontend so captured inputs match exactly what the browser sends; fixtures contain no API key. - Layer 1 — backend golden (tests/test_replay_golden.py): replay through the real gateway, assert the SSE event sequence == committed golden. - Layer 2 — full-stack render (frontend/tests/e2e-real-backend): real Next.js + real gateway (replay model) + Chromium; assert the replayed auto-title and follow-up suggestions render. DOM assertions are the gate; visual regression is a local dev gate (CI uploads the render as an artifact). - CI (.github/workflows/replay-e2e.yml): both layers, triggered on EITHER side of the contract (frontend/** or backend gateway/harness/fixtures). * test(e2e): multi-run render-order cross-stack scenario (#3352) Guards the dangerous front-back class where a backend ordering change silently breaks a frontend assumption while both sides' unit tests stay green. Reproduces issue #3352: backend list_by_thread returns runs newest-first (#2932) and the frontend prepended per-run pages, inverting chronological order once the checkpoint no longer held the older messages. - tests/seed_runs_router.py: test-only seeder, mounted on the replay gateway only when DEERFLOW_ENABLE_TEST_SEED=1 (never in the production app). Seeds a thread with >=2 runs + per-run message events and no checkpoint -- the #3352 precondition -- so the frontend per-run reload path is the sole source of truth and the prepend inversion is observable. - frontend/tests/e2e-real-backend/multi-run-order.spec.ts: drives the real frontend against the real gateway, asserts the first run renders above the second. Reverting the #3354 fix turns it red. - replay-e2e.yml: trigger on the new replay test-infra paths. - docs: REPLAY_E2E.md cross-stack scenario section. * test(e2e): address Copilot review on the replay harness - Fix stale recorder references (scripts/record_traces.py -> scripts/record_gateway.py + scripts/build_fixture_from_jsonl.py) in replay_provider.py, test_replay_golden.py, _replay_fixture.py. - MODE_CONTEXT['ultra']: thinking_enabled False -> True, mirroring the frontend's `context.mode !== 'flash'` (hooks.ts). It did not affect the hashed input (Layer 1 golden still green), but the table now matches the real frontend context it claims to mirror. - replay_provider.py docstring: stop claiming memory is recorded-enabled; the replay config disables memory/summarization for determinism (title stays, as an in-graph deterministic call). - record_gateway.py / run_replay_gateway.py: override DEER_FLOW_HOME instead of setdefault, so an outer value can't leak into the hermetic harness. - record_gateway.py: clear error when DEERFLOW_RECORD_OUT is unset (was a bare KeyError). - playwright.record.config.ts: forward OPENAI_*/DEERFLOW_RECORD_OUT only when set, so the gateway raises a clear 'missing env' error instead of getting ''. * test(e2e): address Copilot review round 2 - seed_runs_router.py: constrain SeedMessage.role to Literal['human','ai'] so a bad value is a clean 422 at the boundary instead of a 500 (KeyError on _EVENT_TYPE). - record-write-read-file.spec.ts: waitForCaptureStable now throws on timeout instead of returning the last count, so a truncated/partial recording can't pass silently. - real-backend-render.spec.ts: guard the suggestions JSON.parse; a bracket-prefixed non-JSON turn falls back to '' so the existing not.toBe('') assertion fails clearly instead of a generic parse throw.
74 lines
3.1 KiB
Python
74 lines
3.1 KiB
Python
"""Start a hermetic *replay* gateway for the full-stack (Layer 2) e2e.
|
|
|
|
Builds an ephemeral config that points the model at ``ReplayChatModel`` + a
|
|
recorded fixture, then runs uvicorn — no API key, deterministic. Used as a
|
|
Playwright ``webServer`` (see ``frontend/playwright.real-backend.config.ts``) and
|
|
runnable standalone for debugging::
|
|
|
|
uv run python scripts/run_replay_gateway.py --port 8011
|
|
|
|
``tests/`` is put on the path so the config ``use: replay_provider:ReplayChatModel``
|
|
resolves; ``GATEWAY_CORS_ORIGINS`` is set so the frontend on :3000 can talk to it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
_BACKEND = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(_BACKEND))
|
|
sys.path.insert(0, str(_BACKEND / "tests")) # replay_provider + build_config_yaml live here
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--port", type=int, default=8011)
|
|
parser.add_argument("--fixture", default=str(_BACKEND / "tests" / "fixtures" / "replay" / "write_read_file.ultra.json"))
|
|
parser.add_argument("--cors", default="http://localhost:3000")
|
|
args = parser.parse_args()
|
|
|
|
from _replay_fixture import REPLAY_MODEL_BLOCK, build_config_yaml, prepare_hermetic_extras
|
|
|
|
home = Path(tempfile.mkdtemp(prefix="replay-gw-"))
|
|
cfg = home / "config.yaml"
|
|
cfg.write_text(build_config_yaml(model_block=REPLAY_MODEL_BLOCK, home=home), encoding="utf-8")
|
|
|
|
# Override (not setdefault): the replay gateway must be hermetic, so an outer
|
|
# DEER_FLOW_HOME can't leak in and shift prompt-affecting paths/skills.
|
|
os.environ["DEER_FLOW_HOME"] = str(home)
|
|
os.environ["DEER_FLOW_CONFIG_PATH"] = str(cfg)
|
|
os.environ["DEER_FLOW_EXTENSIONS_CONFIG_PATH"] = str(prepare_hermetic_extras(home))
|
|
os.environ["DEERFLOW_REPLAY_FIXTURE"] = args.fixture
|
|
os.environ.setdefault("AUTH_JWT_SECRET", "ci-replay-secret")
|
|
os.environ["GATEWAY_CORS_ORIGINS"] = args.cors
|
|
# Child / dynamic imports (resolve_class) search PYTHONPATH too.
|
|
os.environ["PYTHONPATH"] = os.pathsep.join(p for p in (str(_BACKEND), str(_BACKEND / "tests"), os.environ.get("PYTHONPATH", "")) if p)
|
|
|
|
import uvicorn
|
|
|
|
target: str | object = "app.gateway.app:app"
|
|
# Test-only: attach the run/message seeder used by the multi-run render-order
|
|
# e2e (#3352). Imported from tests/ and mounted here only — never in the
|
|
# production app. Pass the app object (not the import string) so the extra
|
|
# router is registered before uvicorn serves it.
|
|
if os.environ.get("DEERFLOW_ENABLE_TEST_SEED") == "1":
|
|
from seed_runs_router import router as seed_router
|
|
|
|
from app.gateway.app import app as gateway_app
|
|
|
|
gateway_app.include_router(seed_router)
|
|
target = gateway_app
|
|
print("[replay-gw] test-only seed router mounted at /api/test-only/seed-runs", flush=True)
|
|
|
|
print(f"[replay-gw] config={cfg} fixture={args.fixture} cors={args.cors} port={args.port}", flush=True)
|
|
uvicorn.run(target, host="127.0.0.1", port=args.port, log_level="warning")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|