MiaoRuidx 8a78c264b7
fix(runtime): cancel runs across live gateway workers (#4500)
* docs(runtime): design cross-worker cancellation

* fix(runtime): cancel runs across gateway workers

* fix(runtime): harden cross-worker cancellation races

让取消请求与 owner 终态写入通过持久化 CAS 决定先后,保证首次取消 action 在不同 worker 路由下保持一致。\n\n将 heartbeat 收敛为续租后仅发送本地中止信号,并补齐完成竞态与路由重试的回归用例。

* docs(runtime): drop implementation plan from PR

移除仅用于实现过程的跨 worker 取消设计记录,保留 README 和 backend/AGENTS.md 中面向最终行为的文档。

* fix(runtime): preserve local cancel fallback

* test(runtime): adapt worker run manager fakes

* docs(runtime): fix run cancel migration registry

---------

Co-authored-by: MiaoRuidx <12540796+MiaoRuidx@users.noreply.github.com>
2026-07-28 21:45:20 +08:00

75 lines
3.7 KiB
Python

"""ORM model for run metadata."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, Index, String, Text, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class RunRow(Base):
__tablename__ = "runs"
run_id: Mapped[str] = mapped_column(String(64), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
assistant_id: Mapped[str | None] = mapped_column(String(128))
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
status: Mapped[str] = mapped_column(String(20), default="pending")
# "pending" | "running" | "success" | "error" | "timeout" | "interrupted"
operation_kind: Mapped[str] = mapped_column(String(32), nullable=False, default="run", server_default=text("'run'"))
model_name: Mapped[str | None] = mapped_column(String(128))
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text)
stop_reason: Mapped[str | None] = mapped_column(String(50))
# Convenience fields (for listing pages without querying RunEventStore)
message_count: Mapped[int] = mapped_column(default=0)
first_human_message: Mapped[str | None] = mapped_column(Text)
last_ai_message: Mapped[str | None] = mapped_column(Text)
# Token usage (accumulated in-memory by RunJournal, written on run completion)
total_input_tokens: Mapped[int] = mapped_column(default=0)
total_output_tokens: Mapped[int] = mapped_column(default=0)
total_tokens: Mapped[int] = mapped_column(default=0)
llm_call_count: Mapped[int] = mapped_column(default=0)
lead_agent_tokens: Mapped[int] = mapped_column(default=0)
subagent_tokens: Mapped[int] = mapped_column(default=0)
middleware_tokens: Mapped[int] = mapped_column(default=0)
token_usage_by_model: Mapped[dict] = mapped_column(JSON, default=dict, server_default=text("'{}'"))
# Follow-up association
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
# Multi-worker run ownership
owner_worker_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
# A non-owning worker records cancellation here; the owner consumes it
# while renewing its lease. The first action wins.
cancel_action: Mapped[str | None] = mapped_column(String(20), nullable=True)
cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
__table_args__ = (
Index("ix_runs_thread_status", "thread_id", "status"),
Index("ix_runs_lease", "lease_expires_at"),
# Cross-process atomicity guarantee: at most one pending/running run per
# thread. Must live in ORM ``__table_args__`` (not just the migration)
# because the empty-DB bootstrap path runs ``create_all`` + ``stamp head``
# and never executes the migration that also defines this index.
Index(
"uq_runs_thread_active",
"thread_id",
unique=True,
sqlite_where=text("status IN ('pending', 'running')"),
postgresql_where=text("status IN ('pending', 'running')"),
),
)