2026-11-05 16:22:12 +08:00

980 lines
43 KiB
Python

"""End-to-end native SGAR runner for one frozen TB2.1 task."""
from __future__ import annotations
import argparse
import hashlib
import json
import mimetypes
import os
import posixpath
import re
import select
import subprocess
import sys
import time
from pathlib import Path
from typing import Any
from sgar_mvp.scripts.external_substrate_worker import (
DEFAULT_MAX_EMBEDDING_REQUESTS,
)
from sgar_mvp.src.model_media import is_model_media_path, MAX_MODEL_MEDIA_BYTES
from .docker_runtime import DockerTaskRuntime, TaskRuntimeError
from .task_loader import (
CANONICAL_BENCHMARK_ID,
CANONICAL_BENCHMARK_VERSION,
DEFAULT_TASK_ROOT,
load_task,
load_tasks,
)
_PIPELINE_SUCCESS_STATUSES = frozenset({"complete_success", "success_with_warnings"})
def _workspace_inventory_representation_contract(
paths: list[str],
sizes: dict[str, int] | None = None,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
"""Describe path representations without reading or claiming file content."""
textual_application_types = {
"application/csv",
"application/javascript",
"application/json",
"application/sql",
"application/toml",
"application/xml",
"application/x-ndjson",
"application/x-yaml",
}
entries: list[dict[str, Any]] = []
known_text = 0
known_non_text = 0
unknown = 0
extensions: set[str] = set()
for path in paths:
extension = Path(path).suffix.lower()
media_type = mimetypes.guess_type(path)[0]
if extension:
extensions.add(extension)
if media_type is None:
compatibility = "unknown"
unknown += 1
elif media_type.startswith("text/") or media_type in textual_application_types:
compatibility = "compatible"
known_text += 1
else:
compatibility = "incompatible"
known_non_text += 1
entries.append({
"path": path,
**({"byte_size": sizes[path]} if sizes and path in sizes else {}),
"extension": extension,
"media_type": media_type,
"text_reader_compatibility": compatibility,
"content_status": "not_read",
"read_requirement": "read_file_before_using_content",
})
contract = {
"contract_kind": "workspace_file_inventory",
"coverage": "complete_path_list_content_unread",
"inventory_artifact_role": "framework_metadata_only",
"working_directory": "/app",
"authorized_runtime_paths": list(paths),
"tool_binding_rule": (
"The inventory artifact handle names this metadata JSON, not any "
"inventoried file. Bind a Tool file or argument port to an exact literal "
"path from authorized_runtime_paths."
),
"generated_process_path_rule": (
"A generated program receives task paths as arguments and resolves them "
"from its actual process working directory; it must not hard-code /app."
),
"file_count": len(paths),
"known_text_file_count": known_text,
"known_non_text_file_count": known_non_text,
"unknown_media_type_count": unknown,
"extensions": sorted(extensions),
"content_facts_available": False,
**({"known_size_file_count": len(sizes),
"total_known_file_bytes": sum(sizes.values()),
"largest_known_file_bytes": max(sizes.values(), default=0)} if sizes else {}),
}
return contract, entries
def _verified_success(
*,
pipeline_status: str,
verifier: dict[str, Any] | None,
failure: dict[str, Any] | None,
) -> bool:
"""Return the official benchmark outcome, not merely process completion."""
if pipeline_status not in _PIPELINE_SUCCESS_STATUSES or failure is not None:
return False
if not verifier or verifier.get("status") != "ok":
return False
try:
return verifier.get("reward") is not None and float(verifier["reward"]) > 0.0
except (TypeError, ValueError):
return False
def _run_dir(root: Path, task_id: str, trial_id: str) -> Path:
root.mkdir(parents=True, exist_ok=True)
path = root / f"{task_id}__{trial_id}"
if path.exists():
raise RuntimeError(f"run directory already exists: {path}")
path.mkdir(parents=True)
return path
def _write(path: Path, value: Any) -> None:
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _worker_command(
args: argparse.Namespace, run_dir: Path, trial_id: str,
*, method_deadline_monotonic: float | None = None,
) -> list[str]:
return [
sys.executable,
"-m",
"sgar_mvp.scripts.external_substrate_worker",
"--config",
str(Path(args.config).resolve()),
"--query",
args.query,
"--output-dir",
str(run_dir / "native"),
"--report-path",
str(run_dir / "native" / "experiment_report.md"),
"--run-id",
trial_id,
"--trial-id",
trial_id,
"--task-state-manifest",
str(run_dir / "task_state_manifest.json"),
"--network-policy",
"declared" if args.network else "disabled",
*(["--max-generation-requests", str(args.max_generation_requests)]
if args.max_generation_requests is not None else []),
*(["--max-embedding-requests", str(args.max_embedding_requests)]
if args.max_embedding_requests is not None else []),
*(["--method-deadline-monotonic", str(method_deadline_monotonic)]
if method_deadline_monotonic is not None else []),
]
def _instruction_file_paths(instruction: str) -> tuple[str, ...]:
"""Locate explicit path spellings; existence and content remain runtime facts."""
text = re.sub(r"https?://[^\s`\"'<>]+", "", instruction)
quoted = re.findall(r"`([^`]+)`|\"([^\"]+)\"|'([^']+)'", text)
paths = {value for group in quoted for value in group if value.startswith("/")}
text = re.sub(r"`[^`]+`|\"[^\"]+\"|'[^']+'", "", text)
paths.update(match.rstrip(".:") for match in re.findall(
r"(?<![\w:/])(/[^\s`\"'<>|;,()\[\]{}]+)", text,
))
return tuple(sorted({posixpath.normpath(path) for path in paths
if path != "/" and not path.startswith("//")
and not any(character in path for character in "\x00\n\r")}))
def _task_state_manifest(
task: Any,
output_dir: Path,
*,
physical_working_dir: str | None = None,
runtime: DockerTaskRuntime | None = None,
) -> dict[str, Any]:
"""Describe declared inputs and files present in the live task workspace.
This is benchmark-adapter metadata only. It never changes the image's
original files. Declared COPY inputs retain their individual handles;
discovered workspace state is one inventory handle whose paths must be
inspected in the task container before their contents can be used.
"""
output_dir.mkdir(parents=True, exist_ok=True)
dockerfile = task.task_dir / "environment" / "Dockerfile"
workdir = "/workspace"
task_root = posixpath.normpath(str(physical_working_dir or workdir))
records: list[dict[str, str]] = []
if dockerfile.is_file():
for raw in dockerfile.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
work_match = re.match(r"WORKDIR\s+(.+)$", line, re.IGNORECASE)
if work_match:
workdir = posixpath.normpath(work_match.group(1).strip())
continue
copy_match = re.match(r"COPY\s+([^\s]+)\s+([^\s]+)\s*$", line, re.IGNORECASE)
if not copy_match:
continue
source_name, target = copy_match.groups()
source = (task.task_dir / "environment" / source_name).resolve()
try:
source.relative_to((task.task_dir / "environment").resolve())
except ValueError:
continue
if not source.is_file() or (source.suffix.lower() not in {".txt", ".md", ".py", ".v", ".tex", ".js", ".ts", ".json", ".csv", ".sql"}
and not is_model_media_path(source.name)):
continue
target_path = target if target.startswith("/") else posixpath.join(workdir, target)
target_path = posixpath.normpath(target_path)
# Docker treats ``COPY file .`` and a destination that names the
# current WORKDIR as directory copies. Resolve the simple
# single-file form here so the public handle names the actual
# file, rather than the directory that contains it.
destination_is_directory = (
target.endswith("/")
or target in {".", "./"}
or target_path == workdir
)
if destination_is_directory:
target_path = posixpath.join(
target_path, posixpath.basename(source_name.rstrip("/"))
)
# Only expose inputs available under the image's actual final
# working directory. Files copied to build or system locations
# remain image internals and must not be advertised as task input
# handles. This is path-neutral: /workspace, /app, /root, etc.
# are all handled through the same logical /app mapping.
if target_path != task_root and not target_path.startswith(task_root + "/"):
continue
logical_name = posixpath.basename(target_path)
if not logical_name or logical_name in {".", ".."}:
continue
records.append({
"logical_name": logical_name,
# The frozen benchmark package is the authorized source root.
# prepare_task_invocation snapshots it into the run directory
# before any model call; keeping this path outside output_dir
# also preserves the input/output separation invariant.
"source_path": str(source),
# Preserve Docker COPY's source basename. Public input
# descriptors use this name in /app/inputs/<logical_name>/;
# the image target basename is represented by target_path.
"source_name": source.name,
"target_path": target_path,
# The runtime maps this image-local path to the stable SGAR
# logical namespace after inspecting the image WORKDIR.
"logical_target_path": target_path,
})
# Dockerfile COPY is incomplete: a RUN command can create task state.
# Discover files in the already-started working directory. A
# directory's contents are operational state, not a set of independent
# mandatory inputs: exposing every discovered file as a public handle
# would force Planner and Compiler to bind unrelated metadata (including
# binary repository internals) before any Tool can inspect the workspace.
# The benchmark mounts verifier files later outside /app.
inventory_status = "not_requested"
inventory_paths: list[str] = []
inventory_sizes: dict[str, int] = {}
if runtime is not None:
inventory_status = "available"
try:
found = runtime.exec_argv(
["find", ".", "-type", "f", "-printf", "%s\t%p\\0"],
timeout=30,
cwd="/app",
)
# BusyBox find need not implement -printf. Optional size metadata
# must not prevent task startup or change authorized paths.
if not found.get("ok"):
found = runtime.exec_argv(
["find", ".", "-type", "f", "-print0"], timeout=30, cwd="/app",
)
if not found.get("ok"):
inventory_status = "unavailable"
else:
raw_paths = str(found.get("stdout") or "")
declared_targets = {
_logical_task_path(item["target_path"],
physical_working_dir=physical_working_dir)
for item in records
}
for item in raw_paths.split("\x00"):
if not item:
continue
size = None
prefix, separator, suffix = item.partition("\t")
if separator and prefix.isascii() and prefix.isdigit():
size, item = int(prefix), suffix
if not item.startswith("./") or "\n" in item or "\r" in item:
continue
logical = posixpath.normpath("/app/" + item[2:])
if (
logical.startswith("/app/")
and not logical.startswith("/app/inputs/")
and logical not in declared_targets
):
inventory_paths.append(logical)
if size is not None:
inventory_sizes[logical] = size
inventory_paths = sorted(set(inventory_paths))
except (OSError, subprocess.SubprocessError, TaskRuntimeError, UnicodeError):
inventory_status = "unavailable"
# Task instructions can name existing inputs outside WORKDIR. Probe
# those exact files, without scanning their parents or treating missing
# output paths as missing inputs. Keep all ordinary workspace handling.
external_paths = tuple(sorted({
_logical_task_path(path, physical_working_dir=physical_working_dir)
for path in _instruction_file_paths(str(task.instruction))
if not _logical_task_path(path, physical_working_dir=physical_working_dir).startswith("/app/")
and _logical_task_path(path, physical_working_dir=physical_working_dir) != "/app"
}))
verified_external_paths: set[str] = set()
if external_paths:
try:
found = runtime.exec_argv(
["find", *external_paths, "-maxdepth", "0", "-type", "f",
"-printf", "%s\t%p\\0"], timeout=30, cwd="/app",
)
# A nonexistent requested output may make find exit nonzero
# while it still reports real inputs. Parse verified output;
# unsupported -printf can fall back to path-only discovery.
raw = str(found.get("stdout") or "")
if not found.get("ok") and not raw:
found = runtime.exec_argv(
["find", *external_paths, "-maxdepth", "0", "-type", "f", "-print0"],
timeout=30, cwd="/app",
)
raw = str(found.get("stdout") or "")
for item in raw.split("\x00"):
size = None
prefix, separator, suffix = item.partition("\t")
if separator and prefix.isascii() and prefix.isdigit():
size, item = int(prefix), suffix
if item in external_paths:
verified_external_paths.add(item)
inventory_paths.append(item)
if size is not None:
inventory_sizes[item] = size
inventory_paths = sorted(set(inventory_paths))
except (OSError, subprocess.SubprocessError, TaskRuntimeError, UnicodeError):
# This optional discovery must not disable the workspace route.
pass
# Supported model media needs real bytes as well as an inventory path.
# Expose independent optional materials; Planner still chooses which
# inputs a node consumes. Ordinary repository files remain inventory.
media_records = []
source_dir = output_dir.parent / (output_dir.name + ".task-state-source")
source_dir.mkdir(parents=False, exist_ok=False)
declared_media = {_logical_task_path(record["target_path"], physical_working_dir=physical_working_dir): record
for record in records if is_model_media_path(record["source_path"])}
for logical in sorted(set(inventory_paths) | set(declared_media)):
if not is_model_media_path(logical) or any(part in {".git", ".hg", ".svn", ".agents", ".codex"}
for part in Path(logical).parts):
continue
name = "media_" + hashlib.sha256(logical.encode()).hexdigest()[:12] + Path(logical).suffix.lower()
snapshot = source_dir / name
snapshot_options = ({"authorized_paths": verified_external_paths}
if logical in verified_external_paths else {})
if not runtime.snapshot_task_file(logical, snapshot, max_bytes=MAX_MODEL_MEDIA_BYTES,
**snapshot_options):
continue
if logical in declared_media:
declared_media[logical].update(source_path=str(snapshot), source_name=name)
continue
media_records.append({"logical_name": name, "source_path": str(snapshot),
"source_name": name, "target_path": logical, "logical_target_path": logical,
"execution_contract": {"contract_kind": "task_media_snapshot",
"original_task_path": logical, "selection": "optional",
"content_facts_available": True}})
# This manifest is a public input descriptor, not a claim about
# file contents. A later Tool must read an indicated task path
# before the plan may use facts from that file.
source = source_dir / "task_environment_inventory.json"
representation_contract, inventory_entries = (
_workspace_inventory_representation_contract(inventory_paths, inventory_sizes)
)
_write(source, {
"protocol": "sgar-task-container-inventory-v1",
"status": inventory_status,
"working_directory": "/app",
"path_contract": {
"tool_logical_workspace_root": "/app",
**({"task_declared_external_paths": sorted(verified_external_paths),
"external_path_rule": "Exact inventory paths outside /app are image-local task inputs; pass them unchanged in declared Tool path/argument ports."}
if verified_external_paths else {}),
"tool_path_rule": (
"Use /app paths only in declared Tool path or cwd arguments; "
"the runtime maps that logical namespace into the task image."
),
"generated_process_cwd_rule": (
"Generated programs must resolve task files from their actual "
"process working directory (for example os.getcwd() or relative "
"paths) and must not hard-code the logical /app root internally."
),
},
"execution_context": "live_task_runtime",
"instruction_authority": "original_task",
"delivery_note": "This is an execution environment. Follow the original task delivery requirements; a written guide does not substitute for requested runtime changes.",
"total_file_count": len(inventory_paths),
"omitted_file_count": 0,
"captured_text_file_count": 0,
"representation_contract": representation_contract,
"files": inventory_entries,
})
records.append({
"logical_name": "task_environment_inventory",
"source_path": str(source),
"source_name": source.name,
"target_path": "/app/inputs/task_environment_inventory/" + source.name,
"logical_target_path": "/app/inputs/task_environment_inventory/" + source.name,
"execution_contract": representation_contract,
})
records.extend(media_records)
payload = {
"protocol": "sgar-tb21-task-state-manifest-v1",
"records": records,
"inventory_status": inventory_status,
"inventory_file_count": len(inventory_paths),
"final_deliverable_contract": _task_final_deliverable_contract(task),
}
_write(output_dir / "task_state_manifest.json", payload)
return payload
def _logical_task_path(path: str, *, physical_working_dir: str | None) -> str:
"""Map an image path back into SGAR's stable /app namespace."""
normalized = posixpath.normpath(str(path))
physical = posixpath.normpath(str(physical_working_dir or "/app"))
if normalized == physical:
return "/app"
if normalized.startswith(physical + "/"):
return "/app" + normalized[len(physical):]
if normalized == "/app" or normalized.startswith("/app/"):
return normalized
return normalized
def _requested_output_in_clause(prefix: str) -> bool:
"""Recognize requested writes, not historical descriptions of supplied state.
Ambiguous narrative is left to Planner with the unchanged task instruction;
it must not become an authoritative output override in the adapter.
"""
clause = re.split(r"[.!?;]\s+", prefix)[-1].strip()
clause = re.sub(r"^(?:[-*+]\s+|\d+[.)]\s+)", "", clause)
clause = clause.lstrip("*` ")
verbs = r"(?:create|write|save|generate|produce|place|store|deliver|output|implement|put|leave|export|compile|build|convert)"
requested = re.search(
rf"^(?:please\s+)?{verbs}\b|\b(?:must|should|need\s+to|have\s+to)\s+{verbs}\b",
clause, re.IGNORECASE,
)
# A stated task goal can coordinate an analysis with a requested write.
# Still require a delivery action nearest the path so inspected inputs
# and descriptions of supplied state do not acquire output authority.
task_goal = re.match(
r"^(?:(?:the|your)\s+)?(?:goal|objective|task)\s+is\s+to\b",
clause, re.IGNORECASE,
)
if not requested and not task_goal:
return re.search(
r"\b(?:final|output|result|deliverable)\b.*\b(?:must|should)\s+be\s+(?:in|at|under)\b",
clause, re.IGNORECASE,
) is not None
# A later input-acquisition verb starts another relation to the path.
actions = list(re.finditer(
rf"\b({verbs}|read(?:ing)?|inspect(?:ing)?|load(?:ing)?|consum(?:e|ing)|processed\s+(?:by|with))\b",
clause, re.IGNORECASE,
))
return bool(actions) and not actions[-1].group(1).casefold().startswith(
("read", "inspect", "load", "consum", "processed")
)
def _explicit_workspace_output_paths(text: str) -> list[tuple[str, bool]]:
"""Return explicit ``/app`` outputs declared by authoritative task text.
Output scope follows the task's grammar: a path is authoritative when it is
in a Deliverables section, follows a mutation/delivery verb in its clause,
or belongs to an indented file list introduced by such a clause. Merely
being a provided/read input never makes a path a final deliverable.
"""
path_pattern = re.compile(
r"(?<![A-Za-z0-9_.-])(/app(?:/[A-Za-z0-9_.+@-]+)+/?)(?![A-Za-z0-9_.+@/-])"
)
output_noun = re.compile(
r"\b(?:files?|directories|directory|outputs?|results?|artifacts?|deliverables?)\b",
re.IGNORECASE,
)
deliverables_match = re.search(
r"(?ims)^\s*(?:#{1,6}\s*)?\*{0,2}deliverables?\*{0,2}\s*$\s*"
r"(.*?)(?=^\s*(?:#{1,6}\s+|\*\*[A-Za-z][^*]+\*\*\s*$)|\Z)",
text,
)
deliverables_span = (
(deliverables_match.start(1), deliverables_match.end(1))
if deliverables_match
else None
)
declared: list[tuple[str, bool]] = []
output_block_indent: int | None = None
offset = 0
for line in text.splitlines(keepends=True):
visible = line.rstrip("\r\n")
stripped = visible.strip()
indent = len(visible) - len(visible.lstrip())
if output_block_indent is not None and stripped:
is_bullet = bool(re.match(r"^\s*[-*+]\s+", visible))
if indent <= output_block_indent and not is_bullet:
output_block_indent = None
inherited_output_scope = output_block_indent is not None
for match in path_pattern.finditer(visible):
absolute_start = offset + match.start()
in_deliverables = bool(
deliverables_span
and deliverables_span[0] <= absolute_start < deliverables_span[1]
)
prefix = visible[: match.start()]
if not (
in_deliverables
or inherited_output_scope
or _requested_output_in_clause(prefix)
):
continue
raw_path = match.group(1).rstrip(".,;:")
is_directory = raw_path.endswith("/")
path = posixpath.normpath(raw_path)
if path == "/app" or not path.startswith("/app/"):
continue
item = (path, is_directory)
if item not in declared:
declared.append(item)
if (
stripped.endswith(":")
and _requested_output_in_clause(visible)
and output_noun.search(visible)
):
output_block_indent = indent
offset += len(line)
# A parent directory is an obligation, but its explicitly named child files
# are the enforceable bundle members. Keep a directory only when it is the
# sole declared output beneath that path.
filtered: list[tuple[str, bool]] = []
for path, is_directory in declared:
if is_directory and any(
other != path and other.startswith(path.rstrip("/") + "/")
for other, _ in declared
):
continue
filtered.append((path, is_directory))
return filtered
def _task_final_deliverable_contract(task: Any) -> dict[str, Any] | None:
"""Extract explicit workspace delivery without verifier introspection."""
text = str(getattr(task, "instruction", "") or "")
declared = _explicit_workspace_output_paths(text)
absolute_paths = [path for path, _ in declared]
if len(absolute_paths) > 1:
return {
"representation": "bundle",
"format_id": "bundle",
"media_type": "application/x.sgar-artifact-bundle",
"extension": "",
"logical_name": "task-deliverables",
"required_members": [
path.removeprefix("/app/") for path in absolute_paths
],
}
if len(absolute_paths) == 1:
path, is_directory = declared[0]
logical_name = posixpath.basename(path)
if is_directory:
return {
"representation": "directory",
"format_id": "directory",
"media_type": "inode/directory",
"extension": "",
"logical_name": logical_name,
"runtime_path": path,
}
extension = Path(logical_name).suffix.lower()
return {
"representation": "file",
"format_id": extension.removeprefix(".") or "file",
"media_type": "application/octet-stream",
"extension": extension,
"logical_name": logical_name,
"runtime_path": path,
}
# Retain the historical relative-name fallback for tasks that do not name an
# absolute workspace output. Nearby delivery language remains mandatory.
pattern = re.compile(
r"\b([A-Za-z0-9][A-Za-z0-9_.-]*\.(?:vo|tex|pdf|jsonl?|csv|md|markdown|txt|py|js|ts))\b",
re.IGNORECASE,
)
for match in pattern.finditer(text):
line_start = text.rfind("\n", 0, match.start()) + 1
if not _requested_output_in_clause(text[line_start : match.start()]):
continue
logical_name = match.group(1)
extension = Path(logical_name).suffix.lower()
return {
"representation": "file",
"format_id": extension.removeprefix(".") or "file",
"media_type": "application/octet-stream",
"extension": extension,
"logical_name": logical_name,
}
return None
def _effective_agent_timeout(args: argparse.Namespace, task: Any) -> float | None:
"""Resolve the method timeout without changing the benchmark metadata."""
if bool(getattr(args, "no_agent_timeout", False)):
return None
override = getattr(args, "agent_timeout", None)
if override is not None:
value = float(override)
if value <= 0:
raise ValueError("agent_timeout_must_be_positive")
return value
return float(task.agent_timeout_sec)
def run_one(args: argparse.Namespace) -> dict[str, Any]:
task = load_task(Path(args.task_root), args.task_id)
agent_timeout = _effective_agent_timeout(args, task)
trial_id = args.trial_id or f"{task.task_id}-{int(time.time())}"
root = Path(args.output_root).resolve()
run_dir = _run_dir(root, task.task_id, trial_id)
runtime = DockerTaskRuntime(task, trial_id=trial_id, output_dir=run_dir)
setup_started = time.monotonic()
manifest = {
"protocol": "sgar-tb21-trial-manifest-v1",
"experiment_id": args.experiment_id,
"benchmark_id": CANONICAL_BENCHMARK_ID,
"benchmark_version": CANONICAL_BENCHMARK_VERSION,
"task_id": task.task_id,
"trial_id": trial_id,
"task_hash": task.task_hash,
"agent_timeout_sec": agent_timeout,
"agent_timeout_policy": "disabled" if agent_timeout is None else "enforced",
"verifier_timeout_sec": task.verifier_timeout_sec,
"network_policy": "declared" if args.network else "disabled",
"proxy_policy": "sgar-experiment-network-v1",
"max_generation_requests": args.max_generation_requests,
"max_embedding_requests": args.max_embedding_requests,
}
_write(run_dir / "trial_manifest.json", manifest)
container_info: dict[str, Any] = {}
worker: subprocess.Popen[str] | None = None
worker_terminal: dict[str, Any] | None = None
verifier: dict[str, Any] | None = None
method_started = time.monotonic()
method_ended = method_started
failure: dict[str, Any] | None = None
framework_success = False
framework_failure: dict[str, Any] | None = None
method_finished = False
try:
container_info = runtime.start()
_task_state_manifest(
task,
run_dir,
physical_working_dir=runtime.working_dir,
runtime=runtime,
)
state_manifest = json.loads((run_dir / "task_state_manifest.json").read_text())
for record in state_manifest.get("records", []):
source = Path(record["source_path"])
target = f"/app/inputs/{record['logical_name']}/{record['source_name']}"
if source.is_file():
staged = runtime.write_file(target, source)
if not staged.get("ok"):
raise TaskRuntimeError("task_state_input_stage_failed")
# Official task images normally already contain their initial
# environment files. Verify that the logical target resolves to
# the image WORKDIR; if an image omits a copied file, restore the
# frozen task-package bytes into that same container namespace.
target_path = str(record.get("logical_target_path") or "")
logical_target = _logical_task_path(
target_path, physical_working_dir=runtime.working_dir
)
if logical_target.startswith(("/app/", "/tmp/")):
present = runtime.exists(logical_target)
if not present.get("exists", False) and source.is_file():
restored = runtime.write_file(logical_target, source)
if not restored.get("ok"):
raise TaskRuntimeError("task_state_target_restore_failed")
manifest.update({"container": container_info, "setup_seconds": time.monotonic() - setup_started})
_write(run_dir / "trial_manifest.json", manifest)
env = os.environ.copy()
env["PYTHONPATH"] = str(Path.cwd()) + os.pathsep + env.get("PYTHONPATH", "")
env["SGAR_PROJECT_ROOT"] = str(Path.cwd().resolve())
(run_dir / "native").mkdir(parents=True, exist_ok=True)
method_started = time.monotonic()
deadline = None if agent_timeout is None else method_started + agent_timeout
worker = subprocess.Popen(
_worker_command(args, run_dir, trial_id, method_deadline_monotonic=deadline),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=(run_dir / "native" / "worker.stderr.log").open("w", encoding="utf-8"),
text=True,
bufsize=1,
env=env,
)
assert worker.stdin is not None and worker.stdout is not None
timed_out = False
with (run_dir / "rpc_actions.jsonl").open("w", encoding="utf-8") as rpc_log:
while True:
remaining = None if deadline is None else deadline - time.monotonic()
if remaining is not None and remaining <= 0:
timed_out = True
break
ready, _, _ = select.select([worker.stdout], [], [], remaining)
if not ready:
timed_out = True
break
line = worker.stdout.readline()
if not line:
break
payload = json.loads(line)
if payload.get("protocol") == "sgar-external-substrate-action-v1":
result = runtime.handle_rpc(payload)
response = {
"protocol": "sgar-external-substrate-result-v1",
"run_id": payload.get("run_id"),
"trial_id": payload.get("trial_id"),
"action_id": payload.get("action_id"),
"result": result,
}
rpc_log.write(json.dumps({"request": payload, "result": result}, ensure_ascii=False) + "\n")
rpc_log.flush()
worker.stdin.write(json.dumps(response, ensure_ascii=False) + "\n")
worker.stdin.flush()
continue
if payload.get("protocol") == "sgar-external-substrate-terminal-v1":
worker_terminal = payload
_write(run_dir / "worker_terminal.json", payload)
break
method_ended = time.monotonic()
if timed_out:
failure = {
"failure_class": "timeout",
"detail": f"agent_timeout_{agent_timeout}s",
}
if worker.poll() is None:
worker.kill()
worker.wait(timeout=30)
if worker.poll() is None:
worker.stdin.close()
worker.wait(timeout=30)
if worker.returncode != 0 and not timed_out:
failure = {"failure_class": "method_failure", "detail": f"worker_exit_{worker.returncode}"}
if worker_terminal is None:
failure = failure or {
"failure_class": "method_failure",
"detail": "worker_terminal_missing",
}
else:
worker_status = str(worker_terminal.get("status") or "")
worker_result = worker_terminal.get("result")
worker_result = worker_result if isinstance(worker_result, dict) else {}
pipeline_status = str(worker_result.get("pipeline_status") or "")
if worker_status != "completed":
failure = failure or {
"failure_class": "method_failure",
"detail": f"worker_status:{worker_status or 'missing'}",
}
elif pipeline_status not in _PIPELINE_SUCCESS_STATUSES:
failure = failure or {
"failure_class": "method_failure",
"detail": f"pipeline_status:{pipeline_status or 'missing'}",
}
# Freeze execution outcome before the independent verifier phase.
# Verifier transport/setup failure cannot undo an already completed
# method, and an official pass cannot complete a failed method.
framework_failure = failure
framework_success = failure is None
method_finished = True
verifier_started = time.monotonic()
verifier = runtime.verify()
verifier["verifier_seconds"] = time.monotonic() - verifier_started
_write(run_dir / "verifier.json", verifier)
(run_dir / "verifier_stdout.log").write_text(
str(verifier.get("stdout") or ""), encoding="utf-8"
)
(run_dir / "verifier_stderr.log").write_text(
str(verifier.get("stderr") or verifier.get("detail") or ""), encoding="utf-8"
)
if verifier.get("status") != "ok":
failure = {"failure_class": "verifier_failure", "detail": verifier.get("status")}
except (TaskRuntimeError, subprocess.TimeoutExpired, OSError, json.JSONDecodeError) as exc:
failure = {"failure_class": "infrastructure_failure", "detail": f"{type(exc).__name__}:{exc}"}
if not method_finished:
framework_failure = failure
method_ended = time.monotonic()
if worker is not None and worker.poll() is None:
worker.kill()
worker.wait()
finally:
cleanup_started = time.monotonic()
cleanup = runtime.cleanup()
total = time.monotonic() - setup_started
worker_result = (
worker_terminal.get("result")
if isinstance(worker_terminal, dict)
and isinstance(worker_terminal.get("result"), dict)
else {}
)
pipeline_status = str(worker_result.get("pipeline_status") or "")
reward = verifier.get("reward") if verifier else None
verified_success = _verified_success(
pipeline_status=pipeline_status,
verifier=verifier,
failure=failure,
)
result = {
**manifest,
"method_wall_clock_seconds": method_ended - method_started,
"e2e_wall_clock_seconds": total,
"verifier": verifier,
"cleanup": cleanup,
"worker_status": worker_terminal.get("status") if worker_terminal else None,
"pipeline_status": pipeline_status or None,
"verifier_status": verifier.get("status") if verifier else None,
"framework_success": framework_success,
"framework_failure": framework_failure,
"verified_success": verified_success,
# `success` is the benchmark outcome used by batch aggregation. The
# verifier process being runnable is recorded separately above.
"success": verified_success,
"official_reward": reward,
"failure": failure,
"native_run_dir": str(run_dir / "native"),
"rpc_log_path": str(run_dir / "rpc_actions.jsonl"),
}
_write(run_dir / "result.json", result)
return result
def preflight(args: argparse.Namespace) -> dict[str, Any]:
tasks, tree_hash = load_tasks(Path(args.task_root))
selected = tasks if args.task_id in {None, "all"} else [
load_task(Path(args.task_root), args.task_id)
]
records = []
for task in selected:
runtime = DockerTaskRuntime(
task,
trial_id=f"preflight-{task.task_id}",
output_dir=Path(args.output_root),
)
records.append(
{
"task_id": task.task_id,
"image": task.image,
"image_digest": runtime.image_digest(),
"task_hash": task.task_hash,
}
)
return {
"protocol": "sgar-tb21-preflight-v1",
"benchmark_id": CANONICAL_BENCHMARK_ID,
"benchmark_version": CANONICAL_BENCHMARK_VERSION,
"task_tree_sha256": tree_hash,
"task_count": len(records),
"images": records,
}
def run_batch(args: argparse.Namespace) -> int:
manifest = preflight(args)
output_root = Path(args.output_root).resolve()
output_root.mkdir(parents=True, exist_ok=True)
(output_root / "preflight.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
results = []
for item in manifest["images"][: args.max_tasks or None]:
task_args = argparse.Namespace(**vars(args))
task_args.task_id = item["task_id"]
results.append(run_one(task_args))
summary = {
"protocol": "sgar-tb21-batch-result-v1",
"task_tree_sha256": manifest["task_tree_sha256"],
"results": results,
"succeeded": sum(bool(item.get("success")) for item in results),
"failed": sum(not bool(item.get("success")) for item in results),
}
(output_root / "batch_result.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0 if summary["failed"] == 0 else 1
def main() -> int:
parser = argparse.ArgumentParser(description="Run one SGAR native TB2.1 task")
parser.add_argument("--task-id", required=False)
parser.add_argument("--task-root", default=str(DEFAULT_TASK_ROOT))
parser.add_argument("--output-root", required=False, default="/tmp/sgar-tb21")
parser.add_argument("--config", default="sgar_mvp/config.json")
parser.add_argument("--query", default=None)
parser.add_argument("--trial-id", default=None)
parser.add_argument("--experiment-id", default="sgar-tb21")
parser.add_argument("--network", action="store_true")
parser.add_argument("--max-tasks", type=int, default=None)
parser.add_argument(
"--agent-timeout",
type=float,
default=None,
help="Override the task agent timeout in seconds.",
)
parser.add_argument(
"--no-agent-timeout",
action="store_true",
help="Disable the SGAR method timeout for a diagnostic run.",
)
parser.add_argument("--max-generation-requests", type=int, default=None)
parser.add_argument(
"--max-embedding-requests",
type=int,
default=DEFAULT_MAX_EMBEDDING_REQUESTS,
help="Optional diagnostic cap on per-trial embedding requests.",
)
parser.add_argument("inventory", nargs="?", choices=("inventory",))
args = parser.parse_args()
if args.inventory:
tasks, tree_hash = load_tasks(Path(args.task_root))
print(json.dumps({"tasks": len(tasks), "task_tree_sha256": tree_hash}, indent=2))
return 0
if args.task_id in {None, "all"}:
return run_batch(args)
if not args.task_id:
parser.error("--task-id is required for a task run")
if args.query is None:
args.query = load_task(Path(args.task_root), args.task_id).instruction
result = run_one(args)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["success"] else 1
if __name__ == "__main__":
raise SystemExit(main())