mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
35023bb908
* feat(scheduler): create conversation schedules and support agent self-stop * fix(scheduler): validate goals and trial requests before dispatch --------- Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
157 lines
8.6 KiB
Python
157 lines
8.6 KiB
Python
"""Offline checks for the real-Gateway synthetic pilot; never calls a provider."""
|
|
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from scripts.benchmark.scheduled_tasks.protocol import assert_arm_installed, fixture, goal_history_evidence, grade_artifact, require_loopback, require_spend_reserve, run_cost, task_prompt
|
|
|
|
|
|
def test_fixture_reader_cannot_open_arbitrary_paths_or_urls():
|
|
assert fixture("inventory")["source"]["document"].startswith("Synthetic")
|
|
for selector in ("../secrets", "file:///tmp/key", "https://example.com", "missing"):
|
|
with pytest.raises(ValueError, match="Unknown synthetic"):
|
|
fixture(selector)
|
|
|
|
|
|
def test_grading_checks_opened_artifact_values_independently_of_success_claim():
|
|
expected = fixture("inventory")["expected"]
|
|
assert grade_artifact(json.dumps(expected), expected)["passed"]
|
|
assert not grade_artifact('{"status":"success","total":11,"items":["Alpha","Beta"]}', expected)["passed"]
|
|
assert not grade_artifact("The goal was satisfied", expected)["passed"]
|
|
assert not grade_artifact('{"total":true,"items":["Alpha","Beta"]}', expected)["passed"]
|
|
assert not grade_artifact("[]", expected)["passed"]
|
|
|
|
|
|
def test_paired_prompt_is_identical_for_both_arms_and_uses_synthetic_source():
|
|
first = task_prompt("corrected-ledger", "report.json")
|
|
second = task_prompt("corrected-ledger", "report.json")
|
|
assert first == second
|
|
assert "read_scheduled_fixture" in first
|
|
assert "subagents" in first
|
|
|
|
|
|
@pytest.mark.parametrize("url", ["https://example.com", "http://127.0.0.1@external.example", "http://127.0.0.1/prefix", "http://127.0.0.1?token=secret"])
|
|
def test_live_target_must_be_a_bare_disposable_loopback_origin(url):
|
|
with pytest.raises(ValueError):
|
|
require_loopback(url)
|
|
|
|
|
|
def test_spend_guard_refuses_over_budget_or_unbounded_admission():
|
|
require_spend_reserve(0.8, 0.1, 1)
|
|
for spent, reserve, ceiling in ((0.95, 0.1, 1), (0, 0, 1), (0, 0.1, 2), (float("nan"), 0.1, 1)):
|
|
with pytest.raises(ValueError):
|
|
require_spend_reserve(spent, reserve, ceiling)
|
|
|
|
|
|
def test_cost_reads_production_usage_buckets_and_detects_model_deviation():
|
|
plan = {"model_alias": "pilot", "provider_model": "configured-model", "pricing": {"input_per_million": 2, "output_per_million": 4}}
|
|
row = {"model_name": "pilot", "total_input_tokens": 100, "total_output_tokens": 10, "token_usage_by_model": {"configured-model": {"input_tokens": 100, "output_tokens": 10}}}
|
|
assert run_cost(row, plan) == pytest.approx(0.00024)
|
|
unknown = copy.deepcopy(row)
|
|
unknown["token_usage_by_model"]["unapproved-model"] = {"input_tokens": 1}
|
|
with pytest.raises(ValueError, match="unexpected model"):
|
|
run_cost(unknown, plan)
|
|
|
|
|
|
def test_arm_check_cannot_treat_an_uninstalled_goal_as_a_successful_goal_arm():
|
|
assert_arm_installed("plain", {"goal_verdict": None})
|
|
assert_arm_installed("goal", {"scheduled_goal_metadata_present": True, "scheduled_goal_objective": "Write report", "goal_verdict": {"satisfied": True}})
|
|
for missing in ({"goal_verdict": {"satisfied": True}}, {"scheduled_goal_metadata_present": True, "scheduled_goal_objective": "Write report"}):
|
|
with pytest.raises(AssertionError, match="comparison is invalid"):
|
|
assert_arm_installed("goal", missing)
|
|
with pytest.raises(AssertionError, match="plain arm"):
|
|
assert_arm_installed("plain", {"scheduled_goal_metadata_present": True})
|
|
|
|
|
|
def test_budget_evidence_uses_actual_checkpoint_goal_counters_and_no_missing_evidence_default():
|
|
entries = [{"values": {"goal": {"objective": "Produce report", "continuation_count": 0, "max_continuations": 8}}}, {"values": {"goal": None}}]
|
|
evidence = goal_history_evidence(entries, "Produce report")
|
|
assert evidence["within_observed_limits"]
|
|
assert evidence["maximum_observed_continuation_count"] == 0
|
|
assert not goal_history_evidence([], "Produce report")["within_observed_limits"]
|
|
entries[0]["values"]["goal"]["continuation_count"] = 9
|
|
assert not goal_history_evidence(entries, "Produce report")["within_observed_limits"]
|
|
|
|
|
|
def test_completion_requires_host_delivery_and_successful_run():
|
|
from scripts.benchmark.scheduled_tasks.protocol import host_delivery_evidence, independent_completion
|
|
|
|
path = "/mnt/user-data/outputs/report.json"
|
|
events = [{"event_type": "run.delivery", "content": {"satisfied": True, "matched_paths": [path], "by_tool": {"present_files": [path]}}}]
|
|
delivery = host_delivery_evidence(events, path)
|
|
assert delivery["delivered"]
|
|
assert independent_completion(artifact_passed=True, fixture_read=True, delivered=True, run_status="success", stop_reason=None)
|
|
for fields in ({"delivered": False}, {"run_status": "error"}):
|
|
data = {"artifact_passed": True, "fixture_read": True, "delivered": True, "run_status": "success", "stop_reason": None, **fields}
|
|
assert not independent_completion(**data)
|
|
assert not host_delivery_evidence([], path)["delivered"]
|
|
assert not host_delivery_evidence([{"event_type": "llm.ai.response", "content": "I presented the file"}], path)["delivered"]
|
|
events[0]["content"]["by_tool"] = {"write_file": [path]}
|
|
assert not host_delivery_evidence(events, path)["delivered"]
|
|
|
|
|
|
@pytest.mark.parametrize(("delivered", "expected"), [(True, True), (False, False)])
|
|
def test_token_cap_does_not_erase_observed_completion_or_substitute_for_delivery(delivered, expected):
|
|
from scripts.benchmark.scheduled_tasks.protocol import independent_completion
|
|
|
|
assert independent_completion(artifact_passed=True, fixture_read=True, delivered=delivered, run_status="success", stop_reason="token_capped") is expected
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interactive_failure_prevents_another_paid_stage():
|
|
from unittest.mock import AsyncMock
|
|
|
|
from scripts.benchmark.scheduled_tasks.runner import Pilot
|
|
|
|
pilot = object.__new__(Pilot)
|
|
pilot.plan = {"model_alias": "pinned-model"}
|
|
pilot.guard = AsyncMock()
|
|
pilot.request = AsyncMock(return_value={"run_id": "failed-creation"})
|
|
pilot.wait_run = AsyncMock(return_value={"status": "error", "run_id": "failed-creation"})
|
|
with pytest.raises(RuntimeError, match="stop before further paid work"):
|
|
await pilot.user_turn("owned-origin", "Create the exact schedule")
|
|
assert pilot.request.await_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_production_fixture_seed_and_scheduler_launch_install_only_goal_arm_metadata(tmp_path):
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from app.scheduler.service import ScheduledTaskService
|
|
from deerflow.config.database_config import DatabaseConfig
|
|
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
|
|
from deerflow.persistence.scheduled_task_runs import ScheduledTaskRunRepository
|
|
from deerflow.persistence.scheduled_tasks import ScheduledTaskRepository
|
|
from deerflow.persistence.thread_meta.sql import ThreadMetaRepository
|
|
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
|
from scripts.benchmark.scheduled_tasks.runner import Rows
|
|
|
|
await init_engine_from_config(DatabaseConfig(backend="sqlite", sqlite_dir=str(tmp_path)))
|
|
(tmp_path / ".scheduled-benchmark-disposable").write_text("synthetic test only", encoding="utf-8")
|
|
rows = Rows(tmp_path)
|
|
try:
|
|
factory = get_session_factory()
|
|
origin = "synthetic-real-origin"
|
|
await ThreadMetaRepository(factory).create(origin, user_id=DEFAULT_USER_ID)
|
|
due = datetime.now(UTC) + timedelta(seconds=2)
|
|
tasks = [await rows.seed_once("inventory", arm, {"scheduler_min_delay": 1}, due_at=due, origin_thread_id=origin) for arm in ("plain", "goal")]
|
|
assert all(task["origin_thread_id"] == origin for task in tasks)
|
|
captured = []
|
|
|
|
async def launch(**kwargs):
|
|
captured.append(kwargs)
|
|
return {"run_id": f"fake-model-run-{len(captured)}", "thread_id": kwargs["thread_id"]}
|
|
|
|
service = ScheduledTaskService(task_repo=ScheduledTaskRepository(factory), task_run_repo=ScheduledTaskRunRepository(factory), launch_run=launch, poll_interval_seconds=1, lease_seconds=120, max_concurrent_runs=3)
|
|
await service.run_once(now=due + timedelta(seconds=1))
|
|
by_task = {entry["metadata"]["scheduled_task_id"]: entry["metadata"] for entry in captured}
|
|
assert "scheduled_goal_objective" not in by_task[tasks[0]["id"]]
|
|
assert by_task[tasks[1]["id"]]["scheduled_goal_objective"] == tasks[1]["goal_objective"]
|
|
with pytest.raises(AssertionError):
|
|
assert_arm_installed("goal", {"scheduled_goal_metadata_present": True, "scheduled_goal_objective": tasks[1]["goal_objective"], "goal_verdict": None})
|
|
finally:
|
|
await rows.close()
|
|
await close_engine()
|