mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-01 08:00:40 +00:00
* fix(console): tolerate malformed usage counters and pricing * docs(routers): record console counter tolerance in the router contract * fix(console): reject stored counters beyond the float range A JSON integer is not bounded by the float range: 10**400 parses as an exact int, so _token_count passed it through and _token_cost then raised OverflowError on the division back to float, taking /usage, /stats and /runs down with a 500 - the same failure mode the rest of this PR removes for every other junk shape. Reject counts whose float conversion is not finite inside the existing try, so they coerce to 0 and the run falls back to its legacy totals. 10**30 stays usable. --------- Co-authored-by: gouyu-hou <gouyu-hou@users.noreply.github.com> Co-authored-by: gouyu-hou <213296172+gouyu-hou@users.noreply.github.com>
611 lines
29 KiB
Python
611 lines
29 KiB
Python
"""Tests for the console router (cross-thread observability endpoints).
|
|
|
|
Covers:
|
|
1. /api/console/stats — headline counters
|
|
2. /api/console/runs — cross-thread listing, thread-title join, pagination, status filter
|
|
3. /api/console/usage — daily zero-filled buckets + per-model breakdown (incl. legacy fallback)
|
|
4. user scoping — rows filtered when the request resolves to a user
|
|
5. 503 when no SQL session factory is available (memory backend)
|
|
|
|
Uses a real temp-file SQLite database (NullPool, so seeding in one event loop
|
|
and serving TestClient requests in another never share a connection).
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
from _router_auth_helpers import make_authed_test_app
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
from app.gateway.routers import console
|
|
from deerflow.persistence.base import Base
|
|
from deerflow.persistence.run.model import RunRow
|
|
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
|
|
|
# Pinned to noon UTC so hour-level seed offsets (NOW - 1h, NOW - 2h) never cross
|
|
# the midnight boundary into the previous calendar day, which would otherwise
|
|
# make "today" rows bucket into yesterday whenever the suite runs within a few
|
|
# hours of UTC midnight. The client fixture freezes the console router's
|
|
# `datetime.now` to this same instant, keeping the router's view of "today"
|
|
# (and active-run durations) aligned with these seed timestamps.
|
|
NOW = datetime.now(UTC).replace(hour=12, minute=0, second=0, microsecond=0)
|
|
|
|
|
|
class _FrozenDatetime(datetime):
|
|
"""``datetime`` subclass whose ``now()`` returns a fixed instant.
|
|
|
|
Everything else (``combine``, ``replace``, arithmetic, ``isinstance``) is
|
|
inherited unchanged from ``datetime``; only ``now`` is redirected so the
|
|
router derives its day-bucket window and live durations from ``NOW``.
|
|
"""
|
|
|
|
_frozen: datetime | None = None
|
|
|
|
@classmethod
|
|
def now(cls, tz=None):
|
|
if cls._frozen is None: # pragma: no cover - defensive fallback
|
|
return super().now(tz)
|
|
return cls._frozen if tz is None else cls._frozen.astimezone(tz)
|
|
|
|
|
|
def _seed_rows() -> tuple[list[ThreadMetaRow], list[RunRow]]:
|
|
threads = [
|
|
ThreadMetaRow(thread_id="t1", user_id="user-a", display_name="调研鹿角再生"),
|
|
ThreadMetaRow(thread_id="t2", user_id="user-a", display_name="Card assistant chat"),
|
|
]
|
|
runs = [
|
|
RunRow(
|
|
run_id="r1",
|
|
thread_id="t1",
|
|
user_id="user-a",
|
|
status="success",
|
|
model_name="minimax-m2",
|
|
total_tokens=1200,
|
|
total_input_tokens=800,
|
|
total_output_tokens=400,
|
|
token_usage_by_model={"minimax-m2": {"input_tokens": 800, "output_tokens": 400, "total_tokens": 1200, "cache_read_tokens": 500}},
|
|
message_count=4,
|
|
created_at=NOW - timedelta(hours=1),
|
|
updated_at=NOW - timedelta(hours=1) + timedelta(seconds=30),
|
|
),
|
|
RunRow(
|
|
run_id="r2",
|
|
thread_id="t1",
|
|
user_id="user-a",
|
|
status="running",
|
|
model_name="minimax-m2",
|
|
total_tokens=300,
|
|
total_input_tokens=200,
|
|
total_output_tokens=100,
|
|
token_usage_by_model={"minimax-m2": {"input_tokens": 200, "output_tokens": 100, "total_tokens": 300}},
|
|
message_count=1,
|
|
created_at=NOW - timedelta(seconds=60),
|
|
updated_at=NOW - timedelta(seconds=1),
|
|
),
|
|
RunRow(
|
|
run_id="r3",
|
|
thread_id="t2",
|
|
user_id="user-a",
|
|
status="error",
|
|
model_name="gpt-x",
|
|
error="Boom: provider exploded",
|
|
total_tokens=50,
|
|
total_input_tokens=50,
|
|
total_output_tokens=0,
|
|
token_usage_by_model={}, # legacy row → model_name fallback path
|
|
message_count=2,
|
|
created_at=NOW - timedelta(days=1),
|
|
updated_at=NOW - timedelta(days=1) + timedelta(seconds=5),
|
|
),
|
|
RunRow(
|
|
run_id="r4",
|
|
thread_id="t2",
|
|
user_id="user-a",
|
|
status="success",
|
|
model_name="minimax-m2",
|
|
total_tokens=999,
|
|
token_usage_by_model={"minimax-m2": {"input_tokens": 600, "output_tokens": 399, "total_tokens": 999}},
|
|
created_at=NOW - timedelta(days=40), # outside the default usage window
|
|
updated_at=NOW - timedelta(days=40) + timedelta(seconds=10),
|
|
),
|
|
RunRow(
|
|
run_id="r5",
|
|
thread_id="t3", # no threads_meta row → exercises the outer join
|
|
user_id="user-b",
|
|
status="success",
|
|
model_name="qwen",
|
|
total_tokens=70,
|
|
total_input_tokens=40,
|
|
total_output_tokens=30,
|
|
token_usage_by_model={"qwen": {"input_tokens": 40, "output_tokens": 30, "total_tokens": 70}},
|
|
created_at=NOW - timedelta(hours=2),
|
|
updated_at=NOW - timedelta(hours=2) + timedelta(seconds=8),
|
|
),
|
|
RunRow(
|
|
run_id="checkpoint-write-1",
|
|
thread_id="t1",
|
|
user_id="user-a",
|
|
operation_kind="checkpoint_write",
|
|
status="error",
|
|
created_at=NOW - timedelta(minutes=30),
|
|
updated_at=NOW - timedelta(minutes=29),
|
|
),
|
|
]
|
|
return threads, runs
|
|
|
|
|
|
@pytest.fixture()
|
|
def session_factory(tmp_path):
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'console.db'}", poolclass=NullPool)
|
|
sf = async_sessionmaker(engine, expire_on_commit=False)
|
|
|
|
async def _setup() -> None:
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
threads, runs = _seed_rows()
|
|
async with sf() as session:
|
|
session.add_all(threads)
|
|
session.add_all(runs)
|
|
await session.commit()
|
|
|
|
asyncio.run(_setup())
|
|
yield sf
|
|
asyncio.run(engine.dispose())
|
|
|
|
|
|
@pytest.fixture()
|
|
def client(session_factory, monkeypatch):
|
|
monkeypatch.setattr(console, "get_session_factory", lambda: session_factory)
|
|
monkeypatch.setattr(console, "get_current_user", AsyncMock(return_value=None))
|
|
monkeypatch.setattr(console, "list_custom_agents", lambda: [object(), object()])
|
|
# No pricing configured by default; TestPricing patches its own config.
|
|
monkeypatch.setattr(console, "get_app_config", lambda: SimpleNamespace(models=[]))
|
|
# Pin the router's wall-clock to NOW so day-bucketing and durations are
|
|
# independent of when the suite runs (see NOW's docstring).
|
|
_FrozenDatetime._frozen = NOW
|
|
monkeypatch.setattr(console, "datetime", _FrozenDatetime)
|
|
app = make_authed_test_app()
|
|
app.include_router(console.router)
|
|
return TestClient(app)
|
|
|
|
|
|
class TestConsoleStats:
|
|
def test_headline_counters(self, client):
|
|
resp = client.get("/api/console/stats")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["total_runs"] == 5
|
|
assert data["active_runs"] == 1 # r2 running
|
|
assert data["failed_runs"] == 1 # r3 error
|
|
assert data["total_threads"] == 2
|
|
assert data["total_agents"] == 2
|
|
assert data["total_tokens"] == 1200 + 300 + 50 + 999 + 70
|
|
|
|
|
|
class TestConsoleRuns:
|
|
def test_listing_orders_paginates_and_joins_titles(self, client):
|
|
resp = client.get("/api/console/runs", params={"limit": 3})
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["has_more"] is True
|
|
ids = [r["run_id"] for r in data["runs"]]
|
|
assert ids == ["r2", "r1", "r5"] # newest first
|
|
by_id = {r["run_id"]: r for r in data["runs"]}
|
|
assert by_id["r2"]["thread_title"] == "调研鹿角再生"
|
|
assert by_id["r5"]["thread_title"] is None # t3 has no threads_meta row
|
|
# Terminal run: duration from created→updated; active run: live elapsed > 0.
|
|
assert by_id["r1"]["duration_seconds"] == pytest.approx(30.0, abs=1.0)
|
|
assert by_id["r2"]["duration_seconds"] > 0
|
|
|
|
def test_offset_pagination(self, client):
|
|
resp = client.get("/api/console/runs", params={"limit": 3, "offset": 3})
|
|
data = resp.json()
|
|
assert [r["run_id"] for r in data["runs"]] == ["r3", "r4"]
|
|
assert data["has_more"] is False
|
|
|
|
def test_status_filter(self, client):
|
|
resp = client.get("/api/console/runs", params={"status": "error"})
|
|
data = resp.json()
|
|
assert [r["run_id"] for r in data["runs"]] == ["r3"]
|
|
assert data["runs"][0]["error"].startswith("Boom")
|
|
|
|
|
|
class TestConsoleUsage:
|
|
def test_daily_buckets_and_model_breakdown(self, client):
|
|
resp = client.get("/api/console/usage", params={"days": 14})
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["days"]) == 14
|
|
# r4 (40 days old) excluded from the window
|
|
assert data["total_runs"] == 4
|
|
assert data["total_tokens"] == 1200 + 300 + 50 + 70
|
|
# Zero-filled series sums to the window total
|
|
assert sum(d["total_tokens"] for d in data["days"]) == data["total_tokens"]
|
|
yesterday = (NOW - timedelta(days=1)).date().isoformat()
|
|
day_map = {d["date"]: d for d in data["days"]}
|
|
assert day_map[yesterday]["total_tokens"] == 50
|
|
assert data["by_model"]["minimax-m2"]["tokens"] == 1500
|
|
assert data["by_model"]["qwen"]["tokens"] == 70
|
|
# Legacy row (empty token_usage_by_model) falls back to model_name
|
|
assert data["by_model"]["gpt-x"]["tokens"] == 50
|
|
assert data["by_model"]["gpt-x"]["input_tokens"] == 50
|
|
|
|
@pytest.mark.parametrize("legacy_usage", [{}, None], ids=["empty-map", "null-map"])
|
|
@pytest.mark.parametrize("legacy_input_tokens", [0, 50])
|
|
def test_legacy_input_tokens_accumulate_with_model_usage(self, client, session_factory, legacy_usage, legacy_input_tokens):
|
|
async def _update_legacy_run() -> None:
|
|
async with session_factory() as session:
|
|
row = await session.get(RunRow, "r3")
|
|
row.model_name = "minimax-m2"
|
|
row.token_usage_by_model = legacy_usage
|
|
row.total_input_tokens = legacy_input_tokens
|
|
row.total_output_tokens = 50 - legacy_input_tokens
|
|
await session.commit()
|
|
|
|
asyncio.run(_update_legacy_run())
|
|
response = client.get("/api/console/usage", params={"days": 14})
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
model = data["by_model"]["minimax-m2"]
|
|
assert model["input_tokens"] == 1000 + legacy_input_tokens
|
|
assert model["tokens"] == 1550
|
|
assert model["runs"] == 3
|
|
assert model["cache_read_tokens"] == 500
|
|
assert sum(day["input_tokens"] for day in data["days"]) == sum(entry["input_tokens"] for entry in data["by_model"].values())
|
|
|
|
def test_window_excludes_old_rows_but_stats_include_them(self, client):
|
|
usage = client.get("/api/console/usage", params={"days": 7}).json()
|
|
assert all(r != 999 for d in usage["days"] for r in [d["total_tokens"]])
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["total_tokens"] >= 999
|
|
|
|
|
|
def _priced_config(*, cache_hit_price: float | None = 0.8):
|
|
pricing = {"currency": "CNY", "input_per_million": 8, "output_per_million": 32}
|
|
if cache_hit_price is not None:
|
|
pricing["input_cache_hit_per_million"] = cache_hit_price
|
|
return SimpleNamespace(
|
|
models=[
|
|
SimpleNamespace(name="minimax-m2", model="MiniMax-M2", pricing=pricing),
|
|
SimpleNamespace(name="gpt-x", model="gpt-x-1", pricing=None), # explicitly unpriced
|
|
]
|
|
)
|
|
|
|
|
|
# Expected per-run costs at 8 (miss) / 0.8 (hit) / 32 (output) per million:
|
|
# r1: 500 of 800 input tokens were cache hits → 300*8 + 500*0.8 + 400*32 (µ¥)
|
|
_R1_COST_CACHED = 300 * 8e-6 + 500 * 0.8e-6 + 400 * 32e-6 # 0.0156
|
|
_R1_COST_UNCACHED = 800 * 8e-6 + 400 * 32e-6 # 0.0192 (no hit price configured)
|
|
_R2_COST = 200 * 8e-6 + 100 * 32e-6 # 0.0048 (no cache hits recorded)
|
|
_R4_COST = 600 * 8e-6 + 399 * 32e-6 # 0.017568
|
|
|
|
|
|
def _mark_missing_usage(session_factory, run_id: str, *, model: str = "minimax-m2", keep_known_tokens: bool = True) -> None:
|
|
async def mutate():
|
|
async with session_factory() as session:
|
|
row = await session.get(RunRow, run_id)
|
|
usage_map = dict(row.token_usage_by_model or {})
|
|
usage = dict(usage_map.get(model) or {}) if keep_known_tokens else {}
|
|
usage["missing_usage_calls"] = 1
|
|
usage_map[model] = usage
|
|
row.token_usage_by_model = usage_map
|
|
await session.commit()
|
|
|
|
asyncio.run(mutate())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"usage_map",
|
|
[
|
|
{"minimax-m2": {"input_tokens": 800, "output_tokens": 400, "missing_usage_calls": 1}},
|
|
{"minimax-m2": {"missing_usage_calls": 1}},
|
|
{"minimax-m2": {"input_tokens": 800, "output_tokens": 400}, "unpriced-evaluator": {"missing_usage_calls": 1}},
|
|
],
|
|
)
|
|
def test_explicit_missing_usage_rejects_full_run_quote_and_fallback(usage_map):
|
|
price = console._ModelPricing(8, 32, "CNY", 0.8)
|
|
assert console._run_cost({"minimax-m2": price}, model_name="minimax-m2", total_input_tokens=800, total_output_tokens=400, token_usage_by_model=usage_map) is None
|
|
|
|
|
|
def test_zero_missing_usage_marker_keeps_legacy_and_cache_pricing():
|
|
price = console._ModelPricing(8, 32, "CNY", 0.8)
|
|
pricing = {"minimax-m2": price}
|
|
assert console._run_cost(pricing, model_name="minimax-m2", total_input_tokens=800, total_output_tokens=400, token_usage_by_model={}) == pytest.approx(_R1_COST_UNCACHED)
|
|
assert console._run_cost(
|
|
pricing,
|
|
model_name="minimax-m2",
|
|
total_input_tokens=800,
|
|
total_output_tokens=400,
|
|
token_usage_by_model={"minimax-m2": {"input_tokens": 800, "output_tokens": 400, "cache_read_tokens": 500, "missing_usage_calls": 0}},
|
|
) == pytest.approx(_R1_COST_CACHED)
|
|
|
|
|
|
class TestPricing:
|
|
@pytest.mark.parametrize("run_id", ["r1", "r2"])
|
|
def test_missing_usage_keeps_same_day_model_and_totals_unpriced_in_any_row_order(self, client, session_factory, monkeypatch, run_id):
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
_mark_missing_usage(session_factory, run_id)
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["total_cost"] is None
|
|
assert stats["total_tokens"] == 2619
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
by_id = {row["run_id"]: row for row in runs["runs"]}
|
|
assert by_id[run_id]["cost"] is None
|
|
assert by_id["r4"]["cost"] == pytest.approx(_R4_COST)
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["total_cost"] is None
|
|
assert usage["by_model"]["minimax-m2"]["cost"] is None
|
|
today = next(day for day in usage["days"] if day["date"] == NOW.date().isoformat())
|
|
assert today["cost"] is None
|
|
assert today["total_tokens"] == 1570
|
|
assert all(day["cost"] == 0 for day in usage["days"] if day["runs"] == 0)
|
|
|
|
def test_unpriced_missing_model_invalidates_run_total_but_not_complete_other_model_bucket(self, client, session_factory, monkeypatch):
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
_mark_missing_usage(session_factory, "r1", model="unpriced-evaluator", keep_known_tokens=False)
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
assert next(row for row in runs["runs"] if row["run_id"] == "r1")["cost"] is None
|
|
assert client.get("/api/console/stats").json()["total_cost"] is None
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["total_cost"] is None
|
|
assert usage["by_model"]["unpriced-evaluator"]["cost"] is None
|
|
assert usage["by_model"]["minimax-m2"]["cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST)
|
|
|
|
def test_missing_usage_outside_window_does_not_taint_window_cost(self, client, session_factory, monkeypatch):
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
_mark_missing_usage(session_factory, "r4")
|
|
assert client.get("/api/console/stats").json()["total_cost"] is None
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["total_cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST)
|
|
|
|
def test_other_users_missing_usage_does_not_taint_scoped_cost(self, client, session_factory, monkeypatch):
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
monkeypatch.setattr(console, "get_current_user", AsyncMock(return_value="user-a"))
|
|
_mark_missing_usage(session_factory, "r5", model="qwen")
|
|
assert client.get("/api/console/stats").json()["total_cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST + _R4_COST)
|
|
assert client.get("/api/console/usage").json()["total_cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST)
|
|
|
|
def test_unknown_window_total_preserves_complete_other_day_and_model_cost(self, client, session_factory, monkeypatch):
|
|
config = _priced_config(cache_hit_price=None)
|
|
config.models[1].pricing = {"currency": "CNY", "input_per_million": 8, "output_per_million": 32}
|
|
monkeypatch.setattr(console, "get_app_config", lambda: config)
|
|
_mark_missing_usage(session_factory, "r1")
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["total_cost"] is None
|
|
yesterday = next(day for day in usage["days"] if day["date"] == (NOW - timedelta(days=1)).date().isoformat())
|
|
assert yesterday["cost"] == pytest.approx(50 * 8e-6)
|
|
assert usage["by_model"]["gpt-x"]["cost"] == pytest.approx(50 * 8e-6)
|
|
|
|
def test_missing_provider_bucket_keeps_its_legacy_profile_alias_unpriced(self, client, session_factory, monkeypatch):
|
|
config = _priced_config()
|
|
config.models[0].name = "profile-alias"
|
|
monkeypatch.setattr(console, "get_app_config", lambda: config)
|
|
_mark_missing_usage(session_factory, "r1")
|
|
|
|
async def make_legacy_alias():
|
|
async with session_factory() as session:
|
|
row = await session.get(RunRow, "r2")
|
|
row.model_name = "profile-alias"
|
|
row.token_usage_by_model = {}
|
|
await session.commit()
|
|
|
|
asyncio.run(make_legacy_alias())
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["by_model"]["minimax-m2"]["cost"] is None
|
|
assert usage["by_model"]["profile-alias"]["cost"] is None
|
|
|
|
def test_mixed_currencies_disable_cost_reporting(self, client, monkeypatch, caplog):
|
|
monkeypatch.setattr(
|
|
console,
|
|
"get_app_config",
|
|
lambda: SimpleNamespace(
|
|
models=[
|
|
SimpleNamespace(
|
|
name="minimax-m2",
|
|
model="MiniMax-M2",
|
|
pricing={"currency": "CNY", "input_per_million": 8, "output_per_million": 32},
|
|
),
|
|
SimpleNamespace(
|
|
name="gpt-x",
|
|
model="gpt-x-1",
|
|
pricing={"currency": "USD", "input_per_million": 1, "output_per_million": 4},
|
|
),
|
|
]
|
|
),
|
|
)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=console.logger.name):
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["currency"] is None
|
|
assert stats["total_cost"] is None
|
|
# The warning names both offending models so operators can locate the misconfiguration.
|
|
assert any("minimax-m2" in rec.getMessage() and "gpt-x" in rec.getMessage() for rec in caplog.records)
|
|
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["currency"] is None
|
|
assert usage["total_cost"] is None
|
|
assert all(day["cost"] == 0 for day in usage["days"])
|
|
assert all(model["cost"] is None for model in usage["by_model"].values())
|
|
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
assert all(run["cost"] is None for run in runs["runs"])
|
|
|
|
def test_shared_currency_across_models_prices_normally(self, client, monkeypatch):
|
|
"""Multiple priced models on one currency (incl. case variants) must not trip the guard."""
|
|
monkeypatch.setattr(
|
|
console,
|
|
"get_app_config",
|
|
lambda: SimpleNamespace(
|
|
models=[
|
|
SimpleNamespace(
|
|
name="minimax-m2",
|
|
model="MiniMax-M2",
|
|
pricing={"currency": "CNY", "input_per_million": 8, "output_per_million": 32},
|
|
),
|
|
SimpleNamespace(
|
|
name="qwen",
|
|
model="qwen",
|
|
pricing={"currency": "cny", "input_per_million": 8, "output_per_million": 32},
|
|
),
|
|
]
|
|
),
|
|
)
|
|
# qwen's only run (r5) is now priced alongside the minimax runs.
|
|
qwen_cost = 40 * 8e-6 + 30 * 32e-6
|
|
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["currency"] == "CNY"
|
|
# No cache-hit price configured → r1 billed at the miss price.
|
|
assert stats["total_cost"] == pytest.approx(_R1_COST_UNCACHED + _R2_COST + _R4_COST + qwen_cost)
|
|
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["currency"] == "CNY"
|
|
assert usage["by_model"]["qwen"]["cost"] == pytest.approx(qwen_cost)
|
|
|
|
def test_costs_use_cache_hit_price(self, client, monkeypatch):
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["currency"] == "CNY"
|
|
# r3 (gpt-x) and r5 (qwen) are unpriced and excluded.
|
|
assert stats["total_cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST + _R4_COST)
|
|
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["currency"] == "CNY"
|
|
assert usage["total_cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST) # r4 outside window
|
|
assert usage["by_model"]["minimax-m2"]["cost"] == pytest.approx(_R1_COST_CACHED + _R2_COST)
|
|
assert usage["by_model"]["minimax-m2"]["input_tokens"] == 1000
|
|
assert usage["by_model"]["minimax-m2"]["cache_read_tokens"] == 500
|
|
assert usage["by_model"]["gpt-x"]["cost"] is None
|
|
assert sum(d["cost"] for d in usage["days"]) == pytest.approx(_R1_COST_CACHED + _R2_COST)
|
|
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
by_id = {r["run_id"]: r for r in runs["runs"]}
|
|
assert by_id["r1"]["cost"] == pytest.approx(_R1_COST_CACHED)
|
|
assert by_id["r3"]["cost"] is None # unpriced model
|
|
|
|
def test_cache_hits_billed_at_miss_price_without_hit_price(self, client, monkeypatch):
|
|
"""No input_cache_hit_per_million configured → conservative upper bound."""
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config(cache_hit_price=None))
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
by_id = {r["run_id"]: r for r in runs["runs"]}
|
|
assert by_id["r1"]["cost"] == pytest.approx(_R1_COST_UNCACHED)
|
|
|
|
def test_costs_null_without_pricing(self, client):
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["total_cost"] is None
|
|
assert stats["currency"] is None
|
|
usage = client.get("/api/console/usage").json()
|
|
assert usage["total_cost"] is None
|
|
runs = client.get("/api/console/runs").json()
|
|
assert all(r["cost"] is None for r in runs["runs"])
|
|
|
|
|
|
class TestUserScoping:
|
|
def test_rows_filtered_by_resolved_user(self, client, monkeypatch):
|
|
monkeypatch.setattr(console, "get_current_user", AsyncMock(return_value="user-a"))
|
|
stats = client.get("/api/console/stats").json()
|
|
assert stats["total_runs"] == 4 # r5 (user-b) excluded
|
|
assert stats["total_tokens"] == 1200 + 300 + 50 + 999
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
assert all(r["run_id"] != "r5" for r in runs["runs"])
|
|
usage = client.get("/api/console/usage").json()
|
|
assert "qwen" not in usage["by_model"]
|
|
|
|
|
|
class TestNoSqlBackend:
|
|
def test_503_when_memory_backend(self, session_factory, monkeypatch):
|
|
monkeypatch.setattr(console, "get_session_factory", lambda: None)
|
|
monkeypatch.setattr(console, "get_current_user", AsyncMock(return_value=None))
|
|
app = make_authed_test_app()
|
|
app.include_router(console.router)
|
|
c = TestClient(app)
|
|
for path in ("/api/console/stats", "/api/console/runs", "/api/console/usage"):
|
|
resp = c.get(path)
|
|
assert resp.status_code == 503
|
|
assert "SQL database backend" in resp.json()["detail"]
|
|
|
|
|
|
class TestMalformedUsageTolerance:
|
|
"""A JSON column written from provider data must not break the console.
|
|
|
|
``runs.token_usage_by_model`` is a schemaless JSON column, and the router
|
|
already skips buckets that are not dicts and pricing that is not numeric.
|
|
Counter values were the one layer left unguarded: ``int("n/a")`` raised out
|
|
of the request and turned every console endpoint into a 500.
|
|
"""
|
|
|
|
@pytest.mark.parametrize("value", ["n/a", "1.5", None, float("nan"), float("inf"), True, False, -7, [1], {"n": 1}, 10**400])
|
|
def test_token_count_falls_back_to_zero_for_unusable_values(self, value):
|
|
assert console._token_count(value) == 0
|
|
|
|
@pytest.mark.parametrize("value, expected", [(0, 0), (12, 12), ("13", 13), (2.0, 2), (10**30, 10**30)])
|
|
def test_token_count_keeps_usable_values(self, value, expected):
|
|
assert console._token_count(value) == expected
|
|
|
|
def test_usage_report_survives_malformed_token_counters(self, client, session_factory):
|
|
_insert_usage_row(session_factory, "r-junk", {"minimax-m2": {"input_tokens": "n/a", "output_tokens": None, "total_tokens": "1.5", "cache_read_tokens": [1], "missing_usage_calls": "bogus"}})
|
|
resp = client.get("/api/console/usage")
|
|
assert resp.status_code == 200
|
|
# The bucket still exists (the run happened) but contributes no tokens.
|
|
assert resp.json()["by_model"]["minimax-m2"]["tokens"] == 1200 + 300
|
|
|
|
def test_run_cost_ignores_malformed_counters_but_keeps_legacy_fallback(self, session_factory, monkeypatch, client):
|
|
_insert_usage_row(session_factory, "r-junk", {"minimax-m2": {"input_tokens": "n/a", "output_tokens": "n/a"}})
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
runs = client.get("/api/console/runs", params={"limit": 50}).json()
|
|
# Both counters coerce to 0 -> the bucket is skipped, and the run keeps
|
|
# its legacy run-level totals priced against model_name.
|
|
row = next(r for r in runs["runs"] if r["run_id"] == "r-junk")
|
|
assert row["cost"] == pytest.approx(_R1_COST_UNCACHED)
|
|
|
|
def test_run_cost_survives_counters_beyond_float_range(self, session_factory, monkeypatch, client):
|
|
# A JSON integer is not bounded by the float range: 10**400 parses as
|
|
# an exact int, and dividing it by a million raised OverflowError out
|
|
# of /runs. It is junk like "n/a", so it must not reach _token_cost.
|
|
_insert_usage_row(session_factory, "r-huge", {"minimax-m2": {"input_tokens": 10**400, "output_tokens": 0}})
|
|
monkeypatch.setattr(console, "get_app_config", lambda: _priced_config())
|
|
resp = client.get("/api/console/runs", params={"limit": 50})
|
|
assert resp.status_code == 200
|
|
row = next(r for r in resp.json()["runs"] if r["run_id"] == "r-huge")
|
|
assert row["cost"] == pytest.approx(_R1_COST_UNCACHED)
|
|
|
|
def test_non_finite_pricing_is_ignored_instead_of_500(self, session_factory, monkeypatch, client):
|
|
monkeypatch.setattr(
|
|
console,
|
|
"get_app_config",
|
|
lambda: SimpleNamespace(models=[SimpleNamespace(name="minimax-m2", model="MiniMax-M2", pricing={"input_per_million": float("nan"), "output_per_million": 32})]),
|
|
)
|
|
resp = client.get("/api/console/usage")
|
|
assert resp.status_code == 200
|
|
# No usable pricing -> costs stay unreported rather than NaN in the body.
|
|
assert resp.json()["by_model"]["minimax-m2"]["cost"] is None
|
|
|
|
|
|
def _insert_usage_row(session_factory, run_id: str, usage_map: dict) -> None:
|
|
async def insert():
|
|
async with session_factory() as session:
|
|
session.add(
|
|
RunRow(
|
|
run_id=run_id,
|
|
thread_id="t1",
|
|
user_id="user-a",
|
|
status="success",
|
|
model_name="minimax-m2",
|
|
total_tokens=1200,
|
|
total_input_tokens=800,
|
|
total_output_tokens=400,
|
|
token_usage_by_model=usage_map,
|
|
created_at=NOW - timedelta(minutes=5),
|
|
updated_at=NOW - timedelta(minutes=4),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
asyncio.run(insert())
|