mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
4246 lines
167 KiB
Python
4246 lines
167 KiB
Python
"""
|
|
S-GAR MVP — End-to-End Pipeline Entry Point
|
|
=============================================
|
|
Orchestrates the full S-GAR pipeline:
|
|
|
|
User Query → [Routing Model] → [Execution Layer] → Final Delivery
|
|
|
|
Architecture Layers:
|
|
┌──────────────────────────────────────────┐
|
|
│ Routing Model (planner.py + router.py)│
|
|
├──────────────────────────────────────────┤
|
|
│ Execution Layer (executors.py + │
|
|
│ orchestrator.py) │
|
|
├──────────────────────────────────────────┤
|
|
│ Contract Layer (schema.py) │
|
|
└──────────────────────────────────────────┘
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import random
|
|
import asyncio
|
|
import argparse
|
|
import hashlib
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from loguru import logger
|
|
|
|
# Fix SSL cert path for Windows (httpx / openai)
|
|
import certifi
|
|
os.environ["SSL_CERT_FILE"] = certifi.where()
|
|
os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
|
|
|
|
# Direct-script compatibility needs the repository root, never the package
|
|
# directory itself. Adding ``sgar_mvp`` to sys.path makes the same source
|
|
# importable as both ``src.*`` and ``sgar_mvp.src.*`` and splits class identity.
|
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
if PROJECT_ROOT not in sys.path:
|
|
sys.path.insert(0, PROJECT_ROOT)
|
|
|
|
from sgar_mvp.src import terminal_progress
|
|
from sgar_mvp.src.runtime_bootstrap import configure_runtime, prepare_retrieval_environment
|
|
from sgar_mvp.src.model_request_budget import ModelRequestBudget, model_request_budget
|
|
from sgar_mvp.src.runtime_abstraction import bound_pipeline
|
|
from sgar_mvp.src.schema import (
|
|
Manifest, Vector, Utility, ManifestType,
|
|
ExecutionMode, PlannerOutput, RoutingDecision, RoutingMetrics,
|
|
Subtask, SubtaskOutputContract,
|
|
)
|
|
from sgar_mvp.src.capability_cards import (
|
|
CapabilityCard,
|
|
build_capability_consistency_report,
|
|
build_capability_cards,
|
|
)
|
|
from sgar_mvp.src.pipeline_control import SubtaskRevisionRef, canonical_sha256
|
|
from sgar_mvp.src.task_invocation import (
|
|
PreparedTaskInvocation,
|
|
ResolvedTaskRequest,
|
|
TaskInvocationError,
|
|
prepare_task_invocation,
|
|
resolve_task_request,
|
|
)
|
|
from sgar_mvp.src.run_workspace import (
|
|
RunManifestStore,
|
|
RunWorkspaceError,
|
|
collect_run_ledger_evidence,
|
|
create_run_workspace,
|
|
sanitized_run_failure,
|
|
structured_run_failure_from_terminal,
|
|
)
|
|
from sgar_mvp.src.terminal_failure import TerminalFailureEnvelope
|
|
from sgar_mvp.src.production_conformance import run_production_conformance
|
|
from sgar_mvp.src.secret_policy import FormalSecretPolicyError, validate_formal_secret_config
|
|
from sgar_mvp.src.executable_plan import (
|
|
RuntimeCapabilities,
|
|
build_runtime_material_adapter_capabilities,
|
|
)
|
|
from sgar_mvp.src.plan_compiler import (
|
|
ExecutablePlanCompiler,
|
|
PlanCompilationStore,
|
|
)
|
|
from sgar_mvp.src.control_role_policy import (
|
|
CONTROL_ROLE_POLICY_PROTOCOL,
|
|
ControlRoleInvocationPolicyV1,
|
|
load_control_role_policy,
|
|
load_control_model_profile,
|
|
require_policy_matches_model_profile,
|
|
)
|
|
from sgar_mvp.src.local_control_readiness import load_git_control_probe_receipt
|
|
from sgar_mvp.src.release_provider_receipt import (
|
|
RELEASE_PROVIDER_PROBE_ROLES,
|
|
resolve_activated_release_provider_probe_receipt,
|
|
)
|
|
from sgar_mvp.src.resource_runtime import (
|
|
CANONICAL_RESULT_PROTOCOL,
|
|
EXECUTION_WORLD_PROTOCOL,
|
|
RESOURCE_CALL_PROTOCOL,
|
|
RESOURCE_RUNTIME_PROTOCOL,
|
|
ResourceDefinition,
|
|
supported_runtime_kinds,
|
|
)
|
|
from sgar_mvp.src.planner import DEFAULT_PLANNER_MAX_OUTPUT_TOKENS, SGARPlanner
|
|
from sgar_mvp.src.planner_contracts import PlannerGenerationError
|
|
from sgar_mvp.src.router import SGARRouter
|
|
from sgar_mvp.src.orchestrator import DAGOrchestrator, NodeUnrecoverableError
|
|
from sgar_mvp.src.reporter import (
|
|
generate_report,
|
|
upsert_execution_section,
|
|
upsert_model_cost_section,
|
|
upsert_recovery_section,
|
|
)
|
|
from sgar_mvp.src.delivery import extract_deliverables
|
|
from sgar_mvp.src.artifact_lifecycle import (
|
|
ArtifactLifecycleCoordinator,
|
|
ArtifactLifecycleError,
|
|
ArtifactLifecycleStore,
|
|
ContextCommitStore,
|
|
)
|
|
from sgar_mvp.src.evaluation_contracts import EvaluatorPolicy
|
|
from sgar_mvp.src.evaluation_runtime import (
|
|
EvaluationCoordinator,
|
|
EvaluationEventLedger,
|
|
EvaluationRuntimeError,
|
|
StaticEvaluationCoordinator,
|
|
load_evaluator_policy,
|
|
normalize_evaluation_mode,
|
|
resolve_evaluator_model,
|
|
)
|
|
from sgar_mvp.src.resource_loader import (
|
|
load_real_pool_with_index,
|
|
load_resource_index_static,
|
|
)
|
|
from sgar_mvp.src.runtime_requirements import scan_environment, warmup_docker_runtime
|
|
from sgar_mvp.src.model_selection import control_model_index
|
|
from sgar_mvp.src.control_models import (
|
|
ControlModelSelector,
|
|
DEFAULT_SYSTEM_MODEL_CHAIN,
|
|
classify_control_model_exception,
|
|
is_control_model_failover_failure,
|
|
)
|
|
from sgar_mvp.src.model_accounting import (
|
|
BudgetControlError,
|
|
ModelAccountingError,
|
|
ModelPricingCatalog,
|
|
RunCostLedger,
|
|
load_model_cost_policy,
|
|
)
|
|
from sgar_mvp.src.model_transport import (
|
|
ModelTransportCapabilityError,
|
|
SyncModelTransportPort,
|
|
classify_transport_exception,
|
|
create_production_model_transport_bundle,
|
|
production_model_endpoint_identity,
|
|
)
|
|
from sgar_mvp.src.model_response_contracts import (
|
|
CapabilityProbeEvidence,
|
|
ExactCapabilityProbeService,
|
|
ModelResponseContractError,
|
|
normalize_capability_probe_enforcement_policy,
|
|
normalize_structured_response_mode,
|
|
system_role_requirement,
|
|
)
|
|
from sgar_mvp.src.system_role_probe_audit import SystemRoleProbeAuditStore
|
|
from sgar_mvp.src.execution_events import ExecutionEventError, RunExecutionLedger
|
|
from sgar_mvp.src.full_generation import (
|
|
FULL_GENERATION_PROMPT_SHA256,
|
|
FULL_GENERATION_PROMPT_VERSION,
|
|
FullGenerationExecutor,
|
|
)
|
|
from sgar_mvp.src.recovery_control import (
|
|
RecoveryControlError,
|
|
RecoveryEventLedger,
|
|
RecoveryPersistenceError,
|
|
RecoveryPolicy,
|
|
load_recovery_policy,
|
|
resolve_system_full_generation_policy,
|
|
)
|
|
from sgar_mvp.src.retrieval_policy import load_retrieval_policy
|
|
from sgar_mvp.src.retrieval_runtime import (
|
|
AppliedReadyStateCapabilityService,
|
|
FrozenCandidatePoolResult,
|
|
RetrievalCoordinator,
|
|
RetrievalPreparationError,
|
|
RetrievalRuntimeError,
|
|
RetrievalRuntimeIdentity,
|
|
build_retrieval_runtime_identity,
|
|
load_applied_model_ready_state,
|
|
project_retrieval_contract,
|
|
validate_loaded_retrieval_backend,
|
|
)
|
|
from sgar_mvp.src.runtime_policy_context import bind_retrieval_policy_path
|
|
from sgar_mvp.src.embedding_runtime import embedding_request_budget
|
|
from sgar_mvp.src.frozen_candidate_publication import (
|
|
persist_frozen_candidate_pool,
|
|
)
|
|
from sgar_mvp.src.formal_serialization import append_formal_jsonl
|
|
from sgar_mvp.src.public_inputs import InternalMetadataLayoutError
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Configuration
|
|
# ─────────────────────────────────────────────
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
DEFAULT_PIPELINE_BUDGET_USD = 10.0
|
|
DEFAULT_USER_QUERY = (
|
|
"The SQL file Pool/resources/resources/ecommerce_schema.sql contains an e-commerce database schema. "
|
|
"Extract its table names and column definitions, then design a RESTful API specification based on the schema. "
|
|
"Using FastAPI, implement the tables' core CRUD operations and combine the code and API specification in a Markdown technical document."
|
|
)
|
|
|
|
|
|
def _portable_artifact_type(*, extension: str, media_type: str, path_kind: str) -> str:
|
|
if path_kind == "directory":
|
|
return "directory"
|
|
normalized_extension = str(extension or "").strip().lower()
|
|
by_extension = {
|
|
".csv": "csv",
|
|
".json": "json",
|
|
".md": "markdown",
|
|
".markdown": "markdown",
|
|
".txt": "plaintext",
|
|
".xlsx": "xlsx",
|
|
".xls": "xls",
|
|
".docx": "docx",
|
|
".doc": "doc",
|
|
".pptx": "pptx",
|
|
".ppt": "ppt",
|
|
".pdf": "pdf",
|
|
".mp3": "audio",
|
|
".wav": "audio",
|
|
".m4a": "audio",
|
|
".flac": "audio",
|
|
".ogg": "audio",
|
|
".png": "image",
|
|
".jpg": "image",
|
|
".jpeg": "image",
|
|
".webp": "image",
|
|
".v": "code",
|
|
".vo": "file",
|
|
".py": "code",
|
|
".js": "code",
|
|
".ts": "code",
|
|
}
|
|
if normalized_extension in by_extension:
|
|
return by_extension[normalized_extension]
|
|
normalized_media = str(media_type or "").strip().lower()
|
|
if "csv" in normalized_media:
|
|
return "csv"
|
|
if "json" in normalized_media:
|
|
return "json"
|
|
if "markdown" in normalized_media:
|
|
return "markdown"
|
|
if normalized_media.startswith("text/"):
|
|
return "plaintext"
|
|
if normalized_media.startswith("audio/"):
|
|
return "audio"
|
|
if normalized_media.startswith("image/"):
|
|
return "image"
|
|
if normalized_media == "application/pdf":
|
|
return "pdf"
|
|
if "spreadsheet" in normalized_media or "excel" in normalized_media:
|
|
return "xlsx"
|
|
if "wordprocessing" in normalized_media:
|
|
return "docx"
|
|
if "msword" in normalized_media:
|
|
return "doc"
|
|
if "presentation" in normalized_media:
|
|
return "pptx"
|
|
if "powerpoint" in normalized_media:
|
|
return "ppt"
|
|
return "file"
|
|
|
|
|
|
def _portable_public_context_for_subtask(
|
|
subtask: Subtask,
|
|
task_invocation: PreparedTaskInvocation | None,
|
|
) -> tuple[dict[str, Any], ...]:
|
|
"""Project only root-node, evidence-cited inputs into host-free metadata.
|
|
|
|
Downstream nodes consume immutable dependency contracts and runtime artifacts;
|
|
they must not silently regain access to the original query or public inputs.
|
|
"""
|
|
|
|
if task_invocation is None:
|
|
return ()
|
|
semantic_v2 = subtask.semantic_contract_v2
|
|
explicit_public_refs = (
|
|
{
|
|
item.ref
|
|
for item in semantic_v2.authorized_inputs
|
|
if item.source == "public_input"
|
|
}
|
|
if semantic_v2 is not None
|
|
else set()
|
|
)
|
|
if semantic_v2 is None and subtask.depends_on:
|
|
return ()
|
|
cited_ids = {
|
|
str(source_id)
|
|
for requirement in subtask.semantic_requirements
|
|
for source_id in requirement.evidence_source_ids
|
|
}
|
|
descriptors: list[dict[str, Any]] = []
|
|
for item in task_invocation.invocation.public_inputs:
|
|
evidence_id = f"artifact:{item.handle_id}"
|
|
if semantic_v2 is not None:
|
|
if str(item.logical_name) not in explicit_public_refs:
|
|
continue
|
|
elif evidence_id not in cited_ids:
|
|
continue
|
|
descriptors.append(
|
|
{
|
|
"logical_name": item.logical_name,
|
|
"source_name": item.source_name,
|
|
"artifact_type": _portable_artifact_type(
|
|
extension=item.extension,
|
|
media_type=item.media_type,
|
|
path_kind=item.path_kind,
|
|
),
|
|
"sha256": item.content_sha256,
|
|
"coverage_status": "handle_only",
|
|
"original_bytes": item.byte_size,
|
|
"included_bytes": 0,
|
|
"included_content_sha256": None,
|
|
"handle_available": True,
|
|
"content_acquisition_required": (
|
|
str(
|
|
item.execution_contract.get("inventory_artifact_role")
|
|
or ""
|
|
).strip()
|
|
!= "framework_metadata_only"
|
|
),
|
|
**({"container_inventory": item.execution_contract["container_inventory"]}
|
|
if item.execution_contract.get("container_inventory") else {}),
|
|
}
|
|
)
|
|
return tuple(descriptors)
|
|
|
|
|
|
def _authoritative_final_json_schema(
|
|
task_invocation: PreparedTaskInvocation | None,
|
|
) -> dict[str, Any] | None:
|
|
"""Return the one invocation-owned authoritative JSON Schema, if present."""
|
|
|
|
if task_invocation is None:
|
|
return None
|
|
descriptors = [
|
|
item
|
|
for item in task_invocation.invocation.public_context_descriptors
|
|
if str(item.get("kind") or "") == "authoritative_json_schema"
|
|
]
|
|
if not descriptors:
|
|
return None
|
|
if len(descriptors) != 1:
|
|
raise ValueError("authoritative_final_json_schema_identity_invalid")
|
|
raw_schema = descriptors[0].get("schema")
|
|
if not isinstance(raw_schema, Mapping):
|
|
raise ValueError("authoritative_final_json_schema_invalid")
|
|
try:
|
|
canonical = json.loads(
|
|
json.dumps(
|
|
dict(raw_schema),
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
)
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("authoritative_final_json_schema_invalid") from exc
|
|
if not isinstance(canonical, dict):
|
|
raise ValueError("authoritative_final_json_schema_invalid")
|
|
return canonical
|
|
|
|
|
|
def _bind_authoritative_final_schema(
|
|
planner_output: PlannerOutput,
|
|
task_invocation: PreparedTaskInvocation | None,
|
|
) -> PlannerOutput:
|
|
"""Bind the invocation-owned schema to the sole semantic final node."""
|
|
|
|
authoritative_schema = _authoritative_final_json_schema(task_invocation)
|
|
if authoritative_schema is None:
|
|
return planner_output
|
|
final_indexes = [
|
|
index
|
|
for index, subtask in enumerate(planner_output.subtasks)
|
|
if subtask.semantic_contract_v2 is not None
|
|
and subtask.semantic_contract_v2.output.contract_scope == "final_deliverable"
|
|
]
|
|
if len(final_indexes) != 1:
|
|
raise ValueError("authoritative_final_node_identity_invalid")
|
|
final_index = final_indexes[0]
|
|
final_subtask = planner_output.subtasks[final_index]
|
|
if final_subtask.artifact_type.value != "json":
|
|
raise ValueError("authoritative_final_schema_artifact_type_invalid")
|
|
if final_subtask.output_contract is None:
|
|
raise ValueError("authoritative_final_output_contract_missing")
|
|
existing_schema = final_subtask.output_contract.json_schema
|
|
if existing_schema is not None:
|
|
if canonical_sha256(existing_schema) != canonical_sha256(authoritative_schema):
|
|
raise ValueError("authoritative_final_json_schema_conflict")
|
|
return planner_output
|
|
|
|
contract_payload = final_subtask.output_contract.model_dump(mode="json")
|
|
contract_payload["json_schema"] = authoritative_schema
|
|
bound_contract = SubtaskOutputContract.model_validate(contract_payload)
|
|
subtasks = list(planner_output.subtasks)
|
|
subtasks[final_index] = final_subtask.model_copy(
|
|
update={"output_contract": bound_contract}
|
|
)
|
|
return planner_output.model_copy(update={"subtasks": subtasks})
|
|
|
|
|
|
def _bind_task_text_authority(
|
|
planner_output: PlannerOutput,
|
|
task_invocation: PreparedTaskInvocation | None,
|
|
) -> PlannerOutput:
|
|
"""Carry the exact request into every node that declares text-only input.
|
|
|
|
Planner semantics may summarize a request, but a task_text_only node has
|
|
no material edge from which omitted literals can be recovered. The exact
|
|
invocation query is therefore authoritative node input. Material-bound
|
|
nodes retain their deliberately scoped inputs and never receive this text.
|
|
"""
|
|
|
|
if task_invocation is None:
|
|
return planner_output
|
|
request_text = str(task_invocation.invocation.query)
|
|
marker = "[SGAR_AUTHORITATIVE_REQUEST]"
|
|
subtasks: list[Subtask] = []
|
|
changed = False
|
|
for subtask in planner_output.subtasks:
|
|
semantic = subtask.semantic_contract_v2
|
|
if semantic is None or semantic.input_requirement != "task_text_only":
|
|
subtasks.append(subtask)
|
|
continue
|
|
if request_text in semantic.task:
|
|
subtasks.append(subtask)
|
|
continue
|
|
authoritative_task = (
|
|
f"{semantic.task.rstrip()}\n\n{marker}\n{request_text}"
|
|
)
|
|
semantic_payload = semantic.model_dump(mode="python")
|
|
semantic_payload.update({
|
|
"task": authoritative_task,
|
|
"contract_sha256": "",
|
|
})
|
|
bound_semantic = semantic.__class__.model_validate(semantic_payload)
|
|
description = str(subtask.description)
|
|
if request_text not in description:
|
|
description = f"{description.rstrip()}\n\n{marker}\n{request_text}"
|
|
subtasks.append(
|
|
subtask.model_copy(
|
|
update={
|
|
"description": description,
|
|
"semantic_contract_v2": bound_semantic,
|
|
}
|
|
)
|
|
)
|
|
changed = True
|
|
return (
|
|
planner_output.model_copy(update={"subtasks": subtasks})
|
|
if changed
|
|
else planner_output
|
|
)
|
|
|
|
|
|
def _model_api_id_from_raw(raw: dict | None) -> str:
|
|
"""Resolve the exact API model ID declared by one Model manifest."""
|
|
|
|
raw = raw if isinstance(raw, dict) else {}
|
|
type_specific = raw.get("type_specific")
|
|
type_specific = type_specific if isinstance(type_specific, dict) else {}
|
|
model_block = type_specific.get("model")
|
|
model_block = model_block if isinstance(model_block, dict) else {}
|
|
execution = raw.get("execution")
|
|
execution = execution if isinstance(execution, dict) else {}
|
|
return str(
|
|
model_block.get("model_id")
|
|
or execution.get("model_id")
|
|
or execution.get("default_base_model")
|
|
or ""
|
|
).strip()
|
|
|
|
|
|
def _resource_id_for_api_model(
|
|
resource_index: dict[str, dict],
|
|
api_model_id: str,
|
|
) -> str:
|
|
matches = sorted(
|
|
resource_id
|
|
for resource_id, raw in resource_index.items()
|
|
if _model_api_id_from_raw(raw) == str(api_model_id)
|
|
)
|
|
if len(matches) != 1:
|
|
raise ModelResponseContractError("system_model_resource_identity_unresolvable")
|
|
return matches[0]
|
|
|
|
|
|
def _probe_system_role(
|
|
*,
|
|
probe_service: ExactCapabilityProbeService,
|
|
role: str,
|
|
resource_id: str,
|
|
api_model_id: str,
|
|
cost_ledger: RunCostLedger,
|
|
audit_store: SystemRoleProbeAuditStore,
|
|
planner_require_capability_fields: bool = False,
|
|
request_policy: ControlRoleInvocationPolicyV1 | None = None,
|
|
) -> CapabilityProbeEvidence:
|
|
requirement = system_role_requirement(
|
|
role,
|
|
planner_require_capability_fields=planner_require_capability_fields,
|
|
)
|
|
|
|
def abort_probe(exc: BaseException) -> None:
|
|
failure = _system_role_probe_exception_failure(
|
|
exc,
|
|
role=role,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
audit_store.record_aborted(
|
|
role=role,
|
|
resource_id=resource_id,
|
|
model_id=api_model_id,
|
|
requirement=requirement,
|
|
endpoint_identity_sha256=(
|
|
probe_service.transport.endpoint_identity.identity_sha256
|
|
if probe_service.transport is not None
|
|
else "0" * 64
|
|
),
|
|
reason_code=failure.failure_code,
|
|
accounting_reference=getattr(exc, "accounting_reference", None),
|
|
)
|
|
audit_store.terminate(failure)
|
|
|
|
try:
|
|
evidence = probe_service.probe(
|
|
resource_id=resource_id,
|
|
model_id=api_model_id,
|
|
requirement=requirement,
|
|
cost_ledger=cost_ledger,
|
|
request_fields=(
|
|
request_policy.request_fields() if request_policy is not None else None
|
|
),
|
|
request_policy_sha256=(
|
|
request_policy.role_policy_sha256
|
|
if request_policy is not None
|
|
else None
|
|
),
|
|
)
|
|
except (asyncio.CancelledError, KeyboardInterrupt) as exc:
|
|
abort_probe(exc)
|
|
raise
|
|
except Exception as exc:
|
|
abort_probe(exc)
|
|
raise
|
|
audit_store.record_evidence(
|
|
role=role,
|
|
resource_id=resource_id,
|
|
model_id=api_model_id,
|
|
requirement=requirement,
|
|
evidence=evidence,
|
|
)
|
|
return evidence
|
|
|
|
|
|
def _system_role_probe_exception_failure(
|
|
exc: BaseException,
|
|
*,
|
|
role: str,
|
|
run_id: str,
|
|
) -> TerminalFailureEnvelope:
|
|
if isinstance(exc, BudgetControlError):
|
|
responsibility = "budget"
|
|
failure_code = str(getattr(exc, "error_code", "model_cost_limit_reached"))
|
|
elif isinstance(exc, ModelAccountingError):
|
|
responsibility = "framework"
|
|
failure_code = str(
|
|
getattr(exc, "error_code", "model_accounting_failure")
|
|
)
|
|
elif isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt)):
|
|
responsibility = "interrupted"
|
|
failure_code = "system_role_schema_probe_interrupted"
|
|
elif isinstance(exc, (ModelTransportCapabilityError, ModelResponseContractError)):
|
|
responsibility = "framework"
|
|
failure_code = str(
|
|
getattr(exc, "error_code", "system_role_schema_probe_contract_invalid")
|
|
)
|
|
else:
|
|
_retryable, transport_code = classify_transport_exception(exc)
|
|
if transport_code != "provider_non_transport_error":
|
|
responsibility = "infrastructure"
|
|
failure_code = transport_code
|
|
else:
|
|
responsibility = "framework"
|
|
failure_code = "system_role_schema_probe_internal_error"
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility=responsibility,
|
|
failure_stage=f"{role}_schema_probe",
|
|
failure_code=failure_code,
|
|
exception=exc,
|
|
retryable=responsibility == "infrastructure",
|
|
response_received=bool(getattr(exc, "response_received", False)),
|
|
run_id=run_id,
|
|
)
|
|
|
|
|
|
def _system_role_probe_evidence_failure(
|
|
evidence: CapabilityProbeEvidence,
|
|
*,
|
|
role: str,
|
|
run_id: str,
|
|
) -> TerminalFailureEnvelope:
|
|
framework_reason_codes = {
|
|
"capability_probe_transport_not_configured",
|
|
"probe_transport_capability_miswired",
|
|
}
|
|
research_reason_codes = {
|
|
"capability_probe_empty_response",
|
|
"capability_probe_invalid_json_response",
|
|
"capability_probe_schema_invalid_response",
|
|
}
|
|
if evidence.reason_code in framework_reason_codes:
|
|
responsibility = "framework"
|
|
elif evidence.reason_code in research_reason_codes:
|
|
responsibility = "research"
|
|
else:
|
|
responsibility = "infrastructure"
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility=responsibility,
|
|
failure_stage=f"{role}_schema_probe",
|
|
failure_code=evidence.reason_code,
|
|
retryable=evidence.outcome == "transient_failure",
|
|
response_received=evidence.response_sha256 is not None,
|
|
run_id=run_id,
|
|
message_sha256=evidence.message_sha256,
|
|
model_operation_id=str(
|
|
(evidence.accounting_reference or {}).get("operation_id") or ""
|
|
) or None,
|
|
)
|
|
|
|
|
|
def _provider_compatibility_projection(base_url: str) -> dict[str, str]:
|
|
"""Return no provider verdict without endpoint-bound capability evidence."""
|
|
|
|
del base_url
|
|
return {}
|
|
|
|
|
|
def _filter_models_for_provider(
|
|
library: list[Manifest],
|
|
fallback_ids: list[str],
|
|
resource_index: dict[str, dict],
|
|
llm_settings: dict,
|
|
base_url: str,
|
|
) -> tuple[list[Manifest], list[str], dict[str, dict]]:
|
|
"""Preserve recall until endpoint-bound evidence proves incompatibility."""
|
|
|
|
del llm_settings, base_url
|
|
return list(library), list(fallback_ids), dict(resource_index)
|
|
|
|
|
|
def load_config(path: str = "config.json", *, resolve_secrets: bool = True) -> dict | None:
|
|
"""Load runtime settings, preferring environment/.env overrides."""
|
|
full = path if os.path.isabs(path) else os.path.join(SCRIPT_DIR, path)
|
|
if not os.path.exists(full):
|
|
logger.warning("Config not found at configured locator.")
|
|
return None
|
|
with open(full, "r", encoding="utf-8") as f:
|
|
config = json.load(f)
|
|
|
|
# Keep secrets out of config.json: explicit environment variables or the
|
|
# project-root .env file always override a legacy literal llm_key.
|
|
if resolve_secrets:
|
|
api_key = _load_env_key()
|
|
if api_key:
|
|
config["llm_key"] = api_key
|
|
|
|
base_url = _load_env_value("LLM_BASE_URL", "OPENAI_BASE_URL")
|
|
if base_url:
|
|
config.setdefault("llm_settings", {})["base_url"] = base_url.rstrip("/")
|
|
|
|
return config
|
|
|
|
|
|
def _runtime_policy_path(
|
|
config: Mapping[str, Any],
|
|
setting_name: str,
|
|
default_relative_path: str,
|
|
) -> Path:
|
|
runtime_settings = config.get("runtime_settings")
|
|
runtime_settings = runtime_settings if isinstance(runtime_settings, Mapping) else {}
|
|
raw = runtime_settings.get(setting_name, default_relative_path)
|
|
if not isinstance(raw, str) or not raw.strip():
|
|
raise RuntimeError(f"runtime_{setting_name}_invalid")
|
|
candidate = Path(raw).expanduser()
|
|
if not candidate.is_absolute():
|
|
candidate = Path(PROJECT_ROOT) / candidate
|
|
resolved = candidate.resolve()
|
|
project_root = Path(PROJECT_ROOT).resolve()
|
|
if not resolved.is_relative_to(project_root) or not resolved.is_file():
|
|
raise RuntimeError(f"runtime_{setting_name}_invalid")
|
|
return resolved
|
|
|
|
|
|
def _load_env_value(*names: str) -> str:
|
|
"""Resolve each preferred name from process env or project-root .env."""
|
|
dotenv_values: dict[str, str] = {}
|
|
env_path = os.path.join(PROJECT_ROOT, ".env")
|
|
if os.path.exists(env_path):
|
|
with open(env_path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
name, value = line.split("=", 1)
|
|
dotenv_values[name.strip()] = value.strip()
|
|
|
|
# Formal production model traffic has exactly one credential authority.
|
|
# Do not silently fall back to another provider variable: doing so makes
|
|
# readiness depend on unrelated host state and weakens the secret audit.
|
|
for name in names:
|
|
value = os.environ.get(name, "").strip() or dotenv_values.get(name, "")
|
|
if value:
|
|
return value
|
|
return ""
|
|
|
|
|
|
def _load_env_key() -> str:
|
|
"""Read the shared LLM credential without logging it."""
|
|
return _load_env_value("LLM_API_KEY")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
"""Parse CLI arguments for reusable experiment runs."""
|
|
parser = argparse.ArgumentParser(description="Run the S-GAR MVP pipeline.")
|
|
parser.add_argument(
|
|
"--query",
|
|
type=str,
|
|
default=None,
|
|
help="Exact user query text. Use exactly one of --query and --query-file.",
|
|
)
|
|
parser.add_argument(
|
|
"--query-file",
|
|
type=str,
|
|
default=None,
|
|
help="Text file containing the exact user query.",
|
|
)
|
|
parser.add_argument(
|
|
"--query-file-encoding",
|
|
type=str,
|
|
default="utf-8-sig",
|
|
help="Explicit query-file encoding (default: utf-8-sig, strict).",
|
|
)
|
|
parser.add_argument(
|
|
"--input",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=PATH",
|
|
help="Explicit public file or directory input. May be repeated.",
|
|
)
|
|
parser.add_argument(
|
|
"--input-manifest",
|
|
type=str,
|
|
default=None,
|
|
help="JSON input manifest. Relative paths resolve from the manifest directory.",
|
|
)
|
|
parser.add_argument(
|
|
"--public-input-root",
|
|
action="append",
|
|
default=[],
|
|
metavar="PATH",
|
|
help=(
|
|
"Explicit read-only root for file-backed public input. May be repeated; "
|
|
"every manifest, query file, and input must remain in exactly one root."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--request-manifest",
|
|
type=str,
|
|
default=None,
|
|
help="Complete JSON TaskInvocation request manifest.",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
type=str,
|
|
default="config.json",
|
|
help="Path to config JSON. Relative paths are resolved from sgar_mvp/.",
|
|
)
|
|
parser.add_argument(
|
|
"--output-root",
|
|
type=str,
|
|
default=None,
|
|
help="Run root; defaults to runtime_settings.output_root in local config.",
|
|
)
|
|
parser.add_argument(
|
|
"--run-dir",
|
|
type=str,
|
|
default=None,
|
|
help="Use an explicitly named new or empty run directory.",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=str,
|
|
default=None,
|
|
help="Deprecated alias for --run-dir.",
|
|
)
|
|
parser.add_argument(
|
|
"--report-path",
|
|
type=str,
|
|
default=None,
|
|
help="Path for the Markdown execution report. Defaults to <output-dir>/experiment_report.md.",
|
|
)
|
|
parser.add_argument(
|
|
"--log-path",
|
|
type=str,
|
|
default=None,
|
|
help="Path for pipeline log. Defaults to <output-dir>/pipeline.log.",
|
|
)
|
|
parser.add_argument(
|
|
"--network-policy",
|
|
choices=("disabled", "declared"),
|
|
default="disabled",
|
|
help="Tool network policy. Network is disabled unless explicitly declared.",
|
|
)
|
|
parser.add_argument(
|
|
"--planner-variant",
|
|
choices=("baseline", "resource_aware"),
|
|
default="resource_aware",
|
|
help=(
|
|
"Planner mode. resource_aware supplies a bounded, frozen-pool "
|
|
"capability view; cited evidence is revalidated before candidate freeze."
|
|
),
|
|
)
|
|
parser.add_argument('--max-generation-requests', type=int, default=None,
|
|
help='Optional existing pipeline generation-request limit.')
|
|
parser.add_argument('--deadline-sec', type=int, default=None,
|
|
help='Optional existing pipeline deadline in seconds.')
|
|
parser.add_argument(
|
|
"--runtime-authority",
|
|
choices=("git", "release"),
|
|
default=None,
|
|
help=(
|
|
"Runtime authority; defaults to local runtime_settings, then git. git uses the current checkout and applied "
|
|
"ready-state; release enables the sealed release attestation gates."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--source-seal",
|
|
type=str,
|
|
default=None,
|
|
help=(
|
|
"Required hash-sealed source identity for strict dirty-working-tree "
|
|
"evaluation or locally activated validation."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--sealed-local-validation",
|
|
action="store_true",
|
|
help=(
|
|
"Validate a dirty Local checkout only when --source-seal and the "
|
|
"sealed retrieval environment are both present."
|
|
),
|
|
)
|
|
args = parser.parse_args()
|
|
for name in ('max_generation_requests', 'deadline_sec'):
|
|
value = getattr(args, name)
|
|
if value is not None and value <= 0:
|
|
parser.error('--' + name.replace('_', '-') + ' must be positive')
|
|
return args
|
|
|
|
|
|
def resolve_user_query(args: argparse.Namespace) -> str:
|
|
"""Resolve the exact user query without stripping or encoding repair."""
|
|
if args.query is not None:
|
|
return args.query
|
|
if args.query_file:
|
|
try:
|
|
with open(
|
|
args.query_file,
|
|
"r",
|
|
encoding=args.query_file_encoding,
|
|
errors="strict",
|
|
) as f:
|
|
return f.read()
|
|
except (OSError, UnicodeError, LookupError) as exc:
|
|
raise TaskInvocationError("query_file_decode_failed") from exc
|
|
return DEFAULT_USER_QUERY
|
|
|
|
|
|
def diagnose_and_repair_query_text(query: str) -> tuple[str, dict]:
|
|
"""Compatibility diagnostic that never rewrites production input."""
|
|
text = query or ""
|
|
diagnostics = {
|
|
"query_encoding_warning": "\ufffd" in text,
|
|
"repair_applied": False,
|
|
"reason": (
|
|
"unicode_replacement_character_present"
|
|
if "\ufffd" in text
|
|
else ""
|
|
),
|
|
}
|
|
return text, diagnostics
|
|
|
|
|
|
def _public_query_reference(query: str) -> str:
|
|
"""Return a content-addressed public reference without copying query text.
|
|
|
|
The exact Unicode query remains available to the production pipeline in
|
|
memory. Reports and traces intentionally persist only this identity so a
|
|
user-supplied host path or credential-like value cannot become public run
|
|
metadata.
|
|
"""
|
|
|
|
raw = query.encode("utf-8")
|
|
return (
|
|
"<task-query "
|
|
f"sha256={canonical_sha256(query)} utf8_bytes={len(raw)}>"
|
|
)
|
|
|
|
|
|
def _public_run_locator(path: str, *, run_dir: str) -> str:
|
|
"""Project a run-owned path to a host-free logical locator."""
|
|
|
|
candidate = Path(path).resolve()
|
|
root = Path(run_dir).resolve()
|
|
try:
|
|
relative = candidate.relative_to(root)
|
|
except ValueError:
|
|
return f"artifact:{canonical_sha256(str(candidate))}"
|
|
return relative.as_posix()
|
|
|
|
|
|
def _resolve_task_request_arguments(
|
|
args: argparse.Namespace,
|
|
*,
|
|
run_dir: Path | None = None,
|
|
) -> ResolvedTaskRequest:
|
|
return resolve_task_request(
|
|
request_manifest=(Path(args.request_manifest) if args.request_manifest else None),
|
|
query=args.query,
|
|
query_file=(Path(args.query_file) if args.query_file else None),
|
|
query_file_encoding=args.query_file_encoding,
|
|
named_inputs=tuple(args.input),
|
|
input_manifest=(Path(args.input_manifest) if args.input_manifest else None),
|
|
allowed_public_input_roots=tuple(
|
|
Path(item) for item in getattr(args, "public_input_root", ())
|
|
),
|
|
project_root=Path(PROJECT_ROOT),
|
|
run_dir=run_dir,
|
|
)
|
|
|
|
|
|
def _protected_contract_guard(subtask: Subtask) -> dict | None:
|
|
artifact_type = subtask.artifact_type.value
|
|
ext = subtask.output_extension or ""
|
|
contract = subtask.output_contract
|
|
produced_files = contract.produced_files if contract is not None else []
|
|
protected = (
|
|
artifact_type in {"code", "json", "csv"}
|
|
or ext.lower() in {".py", ".json", ".csv"}
|
|
or bool(produced_files)
|
|
)
|
|
if not protected:
|
|
return None
|
|
return {
|
|
"id": subtask.id,
|
|
"role": subtask.role,
|
|
"description": subtask.description,
|
|
"expected_output": subtask.expected_output,
|
|
"depends_on": list(subtask.depends_on),
|
|
"artifact_type": artifact_type,
|
|
"output_extension": ext,
|
|
"task_stage": subtask.task_stage.value if subtask.task_stage is not None else None,
|
|
"output_contract": contract.model_dump(mode="json") if contract is not None else None,
|
|
}
|
|
|
|
|
|
def _collect_protected_contracts(planner_output: PlannerOutput) -> dict[str, dict]:
|
|
guards: dict[str, dict] = {}
|
|
for subtask in planner_output.subtasks:
|
|
guard = _protected_contract_guard(subtask)
|
|
if guard is not None:
|
|
guards[subtask.id] = guard
|
|
return guards
|
|
|
|
|
|
def _subtask_satisfies_guard(subtask: Subtask, guard: dict) -> bool:
|
|
guard_type = str(guard.get("artifact_type") or "")
|
|
guard_ext = str(guard.get("output_extension") or "").lower()
|
|
if guard_type and subtask.artifact_type.value == guard_type:
|
|
return True
|
|
if guard_ext and str(subtask.output_extension or "").lower() == guard_ext:
|
|
return True
|
|
guard_contract = guard.get("output_contract") or {}
|
|
guard_files = {
|
|
str(item.get("path_hint"))
|
|
for item in guard_contract.get("produced_files", [])
|
|
if isinstance(item, dict) and item.get("path_hint")
|
|
}
|
|
current_contract = subtask.output_contract
|
|
current_files = set()
|
|
if current_contract is not None:
|
|
current_files = {
|
|
str(item.path_hint)
|
|
for item in current_contract.produced_files
|
|
if item.path_hint
|
|
}
|
|
return bool(guard_files and guard_files.intersection(current_files))
|
|
|
|
|
|
def preserve_replan_contracts(
|
|
planner_output: PlannerOutput,
|
|
original_contracts: dict[str, dict],
|
|
completed_artifacts: dict,
|
|
) -> PlannerOutput:
|
|
"""Prevent graph-level replans from silently downgrading protected artifact contracts."""
|
|
if not original_contracts:
|
|
return planner_output
|
|
|
|
subtasks = list(planner_output.subtasks)
|
|
by_id = {subtask.id: idx for idx, subtask in enumerate(subtasks)}
|
|
completed_ids = set(completed_artifacts.keys())
|
|
changed = False
|
|
|
|
for original_id, guard in original_contracts.items():
|
|
if original_id in completed_ids:
|
|
continue
|
|
if original_id in by_id:
|
|
idx = by_id[original_id]
|
|
current = subtasks[idx]
|
|
updates = {}
|
|
if current.artifact_type.value != guard["artifact_type"]:
|
|
updates["artifact_type"] = guard["artifact_type"]
|
|
if guard.get("output_extension") and current.output_extension != guard["output_extension"]:
|
|
updates["output_extension"] = guard["output_extension"]
|
|
if guard.get("output_contract"):
|
|
updates["output_contract"] = SubtaskOutputContract.model_validate(guard["output_contract"])
|
|
if guard.get("expected_output") and current.expected_output != guard["expected_output"]:
|
|
updates["expected_output"] = guard["expected_output"]
|
|
if guard.get("task_stage") and current.task_stage != guard["task_stage"]:
|
|
updates["task_stage"] = guard["task_stage"]
|
|
if updates:
|
|
subtasks[idx] = current.model_copy(update=updates)
|
|
changed = True
|
|
continue
|
|
|
|
if any(_subtask_satisfies_guard(subtask, guard) for subtask in subtasks):
|
|
continue
|
|
|
|
completion_id = f"{original_id}_contract_completion"
|
|
suffix = 2
|
|
existing_ids = {subtask.id for subtask in subtasks}
|
|
while completion_id in existing_ids:
|
|
completion_id = f"{original_id}_contract_completion_{suffix}"
|
|
suffix += 1
|
|
subtasks.append(
|
|
Subtask(
|
|
id=completion_id,
|
|
role=f"{guard.get('role') or 'contract'}_completion",
|
|
description=(
|
|
"Complete the preserved output contract from the original plan. "
|
|
+ str(guard.get("description") or "")
|
|
),
|
|
expected_output=str(guard.get("expected_output") or guard.get("description") or ""),
|
|
depends_on=list(completed_ids),
|
|
artifact_type=guard["artifact_type"],
|
|
output_extension=guard.get("output_extension") or "",
|
|
task_stage=guard.get("task_stage"),
|
|
output_contract=(
|
|
SubtaskOutputContract.model_validate(guard["output_contract"])
|
|
if guard.get("output_contract")
|
|
else None
|
|
),
|
|
)
|
|
)
|
|
changed = True
|
|
|
|
return planner_output.model_copy(update={"subtasks": subtasks}) if changed else planner_output
|
|
|
|
|
|
def _apply_patch_to_planner_output(planner_output: PlannerOutput, patch) -> PlannerOutput:
|
|
"""Apply a local DAG patch to PlannerOutput without mutating during iteration."""
|
|
task_graph = {node.id: node.model_copy(deep=True) for node in planner_output.subtasks}
|
|
ready_queue: list[str] = []
|
|
processed_ids: set[str] = set()
|
|
DAGOrchestrator.apply_dag_patch(task_graph, ready_queue, processed_ids, patch)
|
|
return planner_output.model_copy(update={"subtasks": list(task_graph.values())})
|
|
|
|
|
|
def _append_jsonl(path: str, event: dict) -> None:
|
|
"""Append a JSONL training trace event without coupling it to logger output."""
|
|
append_formal_jsonl(path, event)
|
|
|
|
|
|
def _atomic_write_json(path: str, payload: object) -> None:
|
|
"""Persist one deterministic JSON artifact without exposing a half-written file."""
|
|
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
temporary = path + ".tmp"
|
|
with open(temporary, "w", encoding="utf-8", newline="\n") as handle:
|
|
json.dump(
|
|
payload,
|
|
handle,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
indent=2,
|
|
default=str,
|
|
)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def _validate_loaded_retrieval_inputs(
|
|
identity: RetrievalRuntimeIdentity,
|
|
library: list[Manifest],
|
|
resource_index: dict[str, dict],
|
|
*,
|
|
policy_path: str | Path | None = None,
|
|
) -> None:
|
|
"""Bind the in-memory pool/index view to the pre-paid immutable identity."""
|
|
|
|
validate_loaded_retrieval_backend(
|
|
identity,
|
|
resource_index=resource_index,
|
|
policy_path=policy_path,
|
|
)
|
|
|
|
library_ids = {item.id for item in library}
|
|
index_ids = set(resource_index)
|
|
eligible_ids = set(identity.eligible_resource_ids)
|
|
if library_ids != eligible_ids or index_ids != eligible_ids:
|
|
raise RetrievalRuntimeError("retrieval_loaded_pool_identity_mismatch")
|
|
if len(library) != len(library_ids):
|
|
raise RetrievalRuntimeError("retrieval_loaded_pool_duplicate_resource_id")
|
|
for manifest in library:
|
|
raw = resource_index.get(manifest.id)
|
|
raw_type_block = raw.get("type") if isinstance(raw, dict) else None
|
|
raw_type = str(
|
|
(raw_type_block.get("resource_type") if isinstance(raw_type_block, dict) else None)
|
|
or (raw.get("resource_type") if isinstance(raw, dict) else None)
|
|
or ""
|
|
)
|
|
if raw_type != manifest.type.value:
|
|
raise RetrievalRuntimeError("retrieval_loaded_manifest_type_mismatch")
|
|
|
|
|
|
def _build_resource_definitions(
|
|
resource_index: dict[str, dict],
|
|
) -> dict[str, ResourceDefinition]:
|
|
"""Project the frozen effective pool once, before any Planner call."""
|
|
|
|
definitions: dict[str, ResourceDefinition] = {}
|
|
for resource_id in sorted(resource_index):
|
|
raw = resource_index[resource_id]
|
|
definition = ResourceDefinition.from_manifest(raw)
|
|
if definition.resource_id != resource_id:
|
|
raise RetrievalRuntimeError("resource_definition_index_identity_mismatch")
|
|
definitions[resource_id] = definition
|
|
return definitions
|
|
|
|
|
|
def _plan_compiler_response_mode(
|
|
raw_model_manifest: dict,
|
|
*,
|
|
configured_mode: str | None = None,
|
|
) -> str:
|
|
"""Require strict schema mode for every machine-parsed system role."""
|
|
|
|
if configured_mode is not None:
|
|
try:
|
|
mode = normalize_structured_response_mode(configured_mode)
|
|
except ModelResponseContractError as exc:
|
|
raise ValueError("formal_system_role_requires_exact_json_schema") from exc
|
|
if mode != "native_strict_schema":
|
|
raise ValueError("formal_system_role_requires_exact_json_schema")
|
|
del raw_model_manifest
|
|
return "native_strict_schema"
|
|
|
|
|
|
def _plan_runtime_capabilities(
|
|
resource_index: dict[str, dict],
|
|
*,
|
|
runtime_image: str,
|
|
) -> RuntimeCapabilities:
|
|
artifact_types = sorted(
|
|
{
|
|
str((raw.get("output_contract") or {}).get("artifact_type") or "unknown")
|
|
for raw in resource_index.values()
|
|
}
|
|
)
|
|
resource_types = sorted(
|
|
{
|
|
str(
|
|
(
|
|
(raw.get("type") or {}).get("resource_type")
|
|
if isinstance(raw.get("type"), dict)
|
|
else None
|
|
)
|
|
or raw.get("resource_type")
|
|
or "Resource"
|
|
)
|
|
for raw in resource_index.values()
|
|
}
|
|
)
|
|
return RuntimeCapabilities(
|
|
runtime_image_id=str(runtime_image),
|
|
supported_runtime_kinds=supported_runtime_kinds(resource_types),
|
|
supported_entrypoint_dispatch_kinds=supported_runtime_kinds(resource_types),
|
|
network_policy="declared_only",
|
|
artifact_contract_support=tuple(artifact_types),
|
|
material_adapter_capabilities=(
|
|
build_runtime_material_adapter_capabilities(resource_types)
|
|
),
|
|
protocol_versions={
|
|
"resource_runtime": RESOURCE_RUNTIME_PROTOCOL,
|
|
"resource_call": RESOURCE_CALL_PROTOCOL,
|
|
"canonical_result": CANONICAL_RESULT_PROTOCOL,
|
|
"execution_world": EXECUTION_WORLD_PROTOCOL,
|
|
},
|
|
)
|
|
|
|
|
|
def _record_frozen_retrieval(
|
|
*,
|
|
trace_path: str,
|
|
output_dir: str,
|
|
run_id: str,
|
|
frozen_result: FrozenCandidatePoolResult,
|
|
) -> str:
|
|
"""Persist the host-free frozen result and its structured lifecycle events."""
|
|
|
|
for attempt in frozen_result.retrieval_attempts:
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "retrieval_attempt",
|
|
"stage": "candidate_pool_preparation",
|
|
"revision": frozen_result.contract_projection.revision.model_dump(mode="json"),
|
|
"attempt": attempt.model_dump(mode="json"),
|
|
},
|
|
)
|
|
publication = persist_frozen_candidate_pool(
|
|
frozen_result=frozen_result,
|
|
run_id=run_id,
|
|
run_dir=output_dir,
|
|
event_writer=lambda event: _append_jsonl(trace_path, dict(event)),
|
|
)
|
|
return publication.candidate_artifact_locator
|
|
|
|
|
|
def _model_liveness_manifest_identity(run_dir: str | Path) -> dict[str, object]:
|
|
"""Project only hashes from published candidate liveness evidence."""
|
|
|
|
candidate_root = Path(run_dir) / "candidate_pools"
|
|
entries: list[dict[str, object]] = []
|
|
if candidate_root.is_dir():
|
|
for path in sorted(candidate_root.glob("*.json")):
|
|
if path.name.endswith(".publication.json"):
|
|
continue
|
|
frozen = FrozenCandidatePoolResult.model_validate_json(
|
|
path.read_text(encoding="utf-8-sig", errors="strict")
|
|
)
|
|
evidence_hashes = [
|
|
item.evidence_sha256 for item in frozen.model_liveness_evidence
|
|
]
|
|
entries.append(
|
|
{
|
|
"revision_sha256": canonical_sha256(frozen.revision),
|
|
"candidate_pool_sha256": (
|
|
frozen.candidate_pool_snapshot.candidate_pool_sha256
|
|
),
|
|
"retrieval_evidence_sha256": frozen.retrieval_evidence_sha256,
|
|
"model_liveness_evidence_sha256": canonical_sha256(
|
|
evidence_hashes
|
|
),
|
|
}
|
|
)
|
|
return {
|
|
"model_liveness_candidate_set_count": len(entries),
|
|
"model_liveness_evidence_sha256": canonical_sha256(entries),
|
|
"model_liveness_candidate_sets": entries,
|
|
}
|
|
|
|
|
|
def _dag_edge_manifest_identity(run_dir: str | Path) -> dict[str, object]:
|
|
"""Project content-free Planner edge identities into the Run Manifest."""
|
|
|
|
trace_path = Path(run_dir) / "trace.jsonl"
|
|
edge_hashes: list[str] = []
|
|
edge_set_hash = canonical_sha256([])
|
|
if trace_path.is_file():
|
|
with trace_path.open("r", encoding="utf-8-sig", errors="strict") as handle:
|
|
for line in handle:
|
|
if not line.strip():
|
|
continue
|
|
event = json.loads(line)
|
|
if event.get("event_type") != "planner_trace":
|
|
continue
|
|
edges = event.get("dag_edge_contracts") or []
|
|
edge_hashes = sorted(
|
|
str(item.get("edge_contract_sha256") or "")
|
|
for item in edges
|
|
if isinstance(item, Mapping)
|
|
and str(item.get("edge_contract_sha256") or "")
|
|
)
|
|
edge_set_hash = canonical_sha256(edges)
|
|
return {
|
|
"dag_edge_contract_count": len(edge_hashes),
|
|
"dag_edge_contract_sha256s": edge_hashes,
|
|
"dag_edge_contracts_sha256": edge_set_hash,
|
|
}
|
|
|
|
|
|
def _planner_attempt_manifest_identity(run_dir: str | Path) -> dict[str, object]:
|
|
"""Project the complete content-free Planner attempt history into the manifest."""
|
|
|
|
evidence_path = Path(run_dir) / "planner_attempts.jsonl"
|
|
records: list[dict[str, object]] = []
|
|
if evidence_path.is_file():
|
|
with evidence_path.open("r", encoding="utf-8-sig", errors="strict") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
if not line.strip():
|
|
continue
|
|
record = json.loads(line)
|
|
if not isinstance(record, dict):
|
|
raise RuntimeError(
|
|
f"planner_attempt_evidence_record_invalid:{line_number}"
|
|
)
|
|
claimed = str(record.get("attempt_evidence_sha256") or "")
|
|
unsigned = dict(record)
|
|
unsigned.pop("attempt_evidence_sha256", None)
|
|
if claimed != canonical_sha256(unsigned):
|
|
raise RuntimeError(
|
|
f"planner_attempt_evidence_hash_mismatch:{line_number}"
|
|
)
|
|
if "raw_response" in record or "provider_reasoning" in record:
|
|
raise RuntimeError("planner_attempt_evidence_contains_private_content")
|
|
records.append(dict(record))
|
|
return {
|
|
"planner_attempt_evidence_protocol": "sgar-planner-attempt-evidence-v1",
|
|
"planner_attempt_count": len(records),
|
|
"planner_attempt_history_sha256": canonical_sha256(records),
|
|
"planner_attempt_file_sha256": (
|
|
hashlib.sha256(evidence_path.read_bytes()).hexdigest()
|
|
if evidence_path.is_file()
|
|
else None
|
|
),
|
|
"planner_attempt_history": records,
|
|
}
|
|
|
|
|
|
def _is_system_execution_failure(message: str) -> bool:
|
|
lowered = str(message or "").lower()
|
|
if "graph_replan_allowed=false" in lowered:
|
|
return True
|
|
if any(
|
|
marker in lowered
|
|
for marker in (
|
|
"failure_category=system_deterministic",
|
|
"failure_category=provider",
|
|
"failure_category=model_content",
|
|
"failure_category=resource_selection",
|
|
"failure_category=budget",
|
|
)
|
|
):
|
|
return True
|
|
return any(
|
|
marker in lowered
|
|
for marker in (
|
|
"validation_target_ambiguous",
|
|
"validation_target_missing",
|
|
"tool_semantic_misuse",
|
|
"binding_ambiguous",
|
|
"tool_path_mapping_error",
|
|
"runner_cwd_error",
|
|
"artifact_dependency_missing",
|
|
"resource_dependency_missing",
|
|
"runtime_profile_unavailable",
|
|
"runtime_warmup_timeout",
|
|
"runtime_image_pull_failed",
|
|
"environment_preflight_failed",
|
|
"dependency_install_blocked",
|
|
"dependency_install_failed",
|
|
"dependency_install_timeout",
|
|
"tool_semantic_failure",
|
|
"tool_semantic_misuse",
|
|
"contract_produced_file_missing",
|
|
"contract_code_extraction_ambiguous",
|
|
"placeholder_content",
|
|
"format_invalid",
|
|
"provider_connection_error",
|
|
"provider_stream_error",
|
|
"model_unavailable",
|
|
"budget_guarded_failure",
|
|
)
|
|
)
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Pipeline
|
|
# ─────────────────────────────────────────────
|
|
|
|
def _session_status_value(session) -> str:
|
|
status = getattr(session, "execution_status", None)
|
|
if status is None:
|
|
return ""
|
|
return getattr(status, "value", str(status))
|
|
|
|
|
|
def _pipeline_recovery_outcome_projection(
|
|
recovery_outcomes: object,
|
|
) -> Mapping[str, object] | None:
|
|
items = sorted(
|
|
(
|
|
dict(item)
|
|
for item in (recovery_outcomes or ())
|
|
if isinstance(item, Mapping)
|
|
),
|
|
key=lambda item: str(item.get("subtask_id") or ""),
|
|
)
|
|
return (
|
|
{
|
|
"protocol": "sgar-recovery-outcomes-v1",
|
|
"subtasks": items,
|
|
}
|
|
if items
|
|
else None
|
|
)
|
|
|
|
|
|
def _successful_pipeline_status(routing_bundles: dict) -> str:
|
|
saw_session = False
|
|
saw_warning = False
|
|
for bundle in routing_bundles.values():
|
|
if not isinstance(bundle, dict):
|
|
continue
|
|
session = bundle.get("routing_session")
|
|
if session is None:
|
|
continue
|
|
saw_session = True
|
|
status = _session_status_value(session)
|
|
if status == "structured_failure" or status == "":
|
|
return _structured_pipeline_failure(
|
|
responsibility="framework",
|
|
failure_stage="pipeline_status",
|
|
failure_code="routing_session_failure_missing_terminal_envelope",
|
|
)
|
|
attempts = getattr(session, "attempts", None) or []
|
|
hard_failures = {
|
|
"operation_kind_missing",
|
|
"operation_misuse",
|
|
"tool_runtime_error",
|
|
"tool_semantic_failure",
|
|
"tool_semantic_misuse",
|
|
"binding_ambiguous",
|
|
"tool_path_type_mismatch",
|
|
}
|
|
has_hard_failure = any(
|
|
str(getattr(attempt, "failure_type", "") or "") in hard_failures
|
|
for attempt in attempts
|
|
)
|
|
has_successful_attempt = any(
|
|
bool(getattr(attempt, "repair_success", False))
|
|
or any(
|
|
str(getattr(step, "status", "") or "") == "success"
|
|
for step in (getattr(attempt, "execution_step_trace", None) or [])
|
|
)
|
|
for attempt in attempts
|
|
)
|
|
if has_hard_failure and not has_successful_attempt:
|
|
return _structured_pipeline_failure(
|
|
responsibility="framework",
|
|
failure_stage="pipeline_status",
|
|
failure_code="routing_attempt_failure_missing_terminal_envelope",
|
|
)
|
|
if status == "success_with_warnings":
|
|
saw_warning = True
|
|
if not saw_session:
|
|
return _structured_pipeline_failure(
|
|
responsibility="framework",
|
|
failure_stage="pipeline_status",
|
|
failure_code="pipeline_finalized_session_missing",
|
|
)
|
|
recovery_outcomes: list[dict[str, object]] = []
|
|
causal_chain: list[dict[str, object]] = []
|
|
for subtask_id in sorted(str(item) for item in routing_bundles):
|
|
bundle = routing_bundles.get(subtask_id)
|
|
recovery = bundle.get("recovery") if isinstance(bundle, Mapping) else None
|
|
if not isinstance(recovery, Mapping):
|
|
continue
|
|
outcome = recovery.get("recovery_outcome")
|
|
if outcome is not None:
|
|
recovery_outcomes.append(
|
|
{"subtask_id": subtask_id, "outcome": str(outcome)}
|
|
)
|
|
for item in recovery.get("causal_chain") or ():
|
|
code = str(item or "")
|
|
if not code:
|
|
continue
|
|
causal_chain.append(
|
|
{
|
|
"sequence": len(causal_chain),
|
|
"kind": "recovered_event",
|
|
"subtask_id": subtask_id,
|
|
"failure_code": code,
|
|
}
|
|
)
|
|
return PipelineTerminalStatus(
|
|
"success_with_warnings" if saw_warning else "complete_success",
|
|
recovery_outcomes=tuple(recovery_outcomes),
|
|
causal_chain=tuple(causal_chain),
|
|
)
|
|
|
|
|
|
class PipelineTerminalStatus(str):
|
|
"""String-compatible pipeline result carrying structured terminal ownership."""
|
|
|
|
def __new__(
|
|
cls,
|
|
value: str,
|
|
*,
|
|
responsibility: str = "research",
|
|
failure_stage: str = "pipeline",
|
|
failure_code: str = "pipeline_structured_failure",
|
|
failure: TerminalFailureEnvelope | None = None,
|
|
primary_failure: TerminalFailureEnvelope | None = None,
|
|
terminal_failure: TerminalFailureEnvelope | None = None,
|
|
recovery_outcomes: tuple[Mapping[str, object], ...] = (),
|
|
causal_chain: tuple[Mapping[str, object], ...] = (),
|
|
) -> "PipelineTerminalStatus":
|
|
instance = str.__new__(cls, value)
|
|
resolved_terminal = terminal_failure or failure or primary_failure
|
|
resolved_primary = primary_failure or failure or terminal_failure
|
|
instance.responsibility = (
|
|
resolved_terminal.responsibility
|
|
if resolved_terminal is not None
|
|
else responsibility
|
|
)
|
|
instance.failure_stage = (
|
|
resolved_terminal.failure_stage
|
|
if resolved_terminal is not None
|
|
else failure_stage
|
|
)
|
|
instance.failure_code = (
|
|
resolved_terminal.failure_code
|
|
if resolved_terminal is not None
|
|
else failure_code
|
|
)
|
|
instance.primary_failure = resolved_primary
|
|
instance.terminal_failure = resolved_terminal
|
|
instance.failure = resolved_terminal
|
|
instance.recovery_outcomes = tuple(
|
|
dict(item) for item in recovery_outcomes
|
|
)
|
|
instance.causal_chain = tuple(dict(item) for item in causal_chain)
|
|
return instance
|
|
|
|
|
|
def _pipeline_terminal_projection(status: object) -> dict[str, object]:
|
|
terminal = getattr(status, "terminal_failure", None) or getattr(
|
|
status, "failure", None
|
|
)
|
|
primary = getattr(status, "primary_failure", None) or terminal
|
|
if not isinstance(primary, TerminalFailureEnvelope) or not isinstance(
|
|
terminal, TerminalFailureEnvelope
|
|
):
|
|
raise TypeError("pipeline_terminal_projection_requires_failures")
|
|
recovery_outcome = _pipeline_recovery_outcome_projection(
|
|
getattr(status, "recovery_outcomes", ())
|
|
)
|
|
causal_chain = tuple(
|
|
dict(item)
|
|
for item in (getattr(status, "causal_chain", ()) or ())
|
|
if isinstance(item, Mapping)
|
|
)
|
|
if not causal_chain:
|
|
causal: list[dict[str, object]] = [
|
|
{
|
|
"sequence": 0,
|
|
"kind": "primary_failure",
|
|
"subtask_id": primary.subtask_id,
|
|
"responsibility": primary.responsibility,
|
|
"failure_stage": primary.failure_stage,
|
|
"failure_code": primary.failure_code,
|
|
"failure_sha256": primary.failure_sha256,
|
|
}
|
|
]
|
|
if terminal.failure_sha256 != primary.failure_sha256:
|
|
causal.append(
|
|
{
|
|
"sequence": 1,
|
|
"kind": "terminal_failure",
|
|
"subtask_id": terminal.subtask_id,
|
|
"responsibility": terminal.responsibility,
|
|
"failure_stage": terminal.failure_stage,
|
|
"failure_code": terminal.failure_code,
|
|
"failure_sha256": terminal.failure_sha256,
|
|
}
|
|
)
|
|
causal_chain = tuple(causal)
|
|
return {
|
|
"status": terminal.run_status,
|
|
"primary_failure": structured_run_failure_from_terminal(primary),
|
|
"terminal_failure": structured_run_failure_from_terminal(terminal),
|
|
"recovery_outcome": recovery_outcome,
|
|
"causal_chain": causal_chain,
|
|
}
|
|
|
|
|
|
def _structured_pipeline_failure(
|
|
*,
|
|
responsibility: str,
|
|
failure_stage: str,
|
|
failure_code: str,
|
|
exception_type: str = "",
|
|
message_sha256: str | None = None,
|
|
response_received: bool = False,
|
|
) -> PipelineTerminalStatus:
|
|
if responsibility not in {
|
|
"framework",
|
|
"infrastructure",
|
|
"research",
|
|
"budget",
|
|
"interrupted",
|
|
}:
|
|
responsibility = "framework"
|
|
failure_stage = "failure_contract"
|
|
failure_code = "pipeline_failure_responsibility_invalid"
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility=responsibility,
|
|
failure_stage=failure_stage,
|
|
failure_code=failure_code,
|
|
exception_type=exception_type,
|
|
message_sha256=message_sha256,
|
|
response_received=response_received,
|
|
)
|
|
return PipelineTerminalStatus(
|
|
"structured_failure",
|
|
responsibility=responsibility,
|
|
failure_stage=failure_stage,
|
|
failure_code=failure_code,
|
|
failure=failure,
|
|
)
|
|
|
|
|
|
def _pipeline_status_from_terminal_failure(
|
|
failure: TerminalFailureEnvelope,
|
|
) -> PipelineTerminalStatus:
|
|
return PipelineTerminalStatus(
|
|
"structured_failure",
|
|
responsibility=failure.responsibility,
|
|
failure_stage=failure.failure_stage,
|
|
failure_code=failure.failure_code,
|
|
failure=failure,
|
|
)
|
|
|
|
|
|
def _execution_terminal_failure_status(execution_summary: dict) -> PipelineTerminalStatus:
|
|
"""Derive ownership only from ResourceRuntime terminal event producers."""
|
|
|
|
counts = dict(execution_summary.get("by_responsibility") or {})
|
|
for responsibility in (
|
|
"framework",
|
|
"interrupted",
|
|
"infrastructure",
|
|
"budget",
|
|
"research",
|
|
):
|
|
if int(counts.get(responsibility) or 0) > 0:
|
|
return _structured_pipeline_failure(
|
|
responsibility=responsibility,
|
|
failure_stage="execution",
|
|
failure_code="resource_execution_terminal_failure",
|
|
)
|
|
return _structured_pipeline_failure(
|
|
responsibility="research",
|
|
failure_stage="pipeline",
|
|
failure_code="pipeline_structured_failure",
|
|
)
|
|
|
|
|
|
def _final_pipeline_log_label(status: str) -> str:
|
|
if status == "complete_success":
|
|
return "S-GAR MVP - Complete (success)"
|
|
if status == "success_with_warnings":
|
|
return "S-GAR MVP - Complete (success_with_warnings)"
|
|
return "S-GAR MVP - Structured Failure"
|
|
|
|
|
|
def _terminal_pipeline_log_lines(status: str) -> list[str]:
|
|
"""Return terminal status lines derived only from the strict pipeline result."""
|
|
return [_final_pipeline_log_label(status)]
|
|
|
|
|
|
def _planner_parse_metadata_with_control_trace(
|
|
planner: SGARPlanner,
|
|
*,
|
|
selected_model: str,
|
|
attempted_models: list[dict],
|
|
) -> dict:
|
|
metadata = dict(getattr(planner, "last_parse_metadata", {}) or {})
|
|
metadata["selected_control_model"] = selected_model
|
|
metadata["attempted_control_models"] = [item.get("model_id") for item in attempted_models]
|
|
metadata["control_model_failover_reasons"] = [
|
|
{
|
|
"model_id": item.get("model_id"),
|
|
"failure_type": item.get("failure_type"),
|
|
"reason": item.get("reason"),
|
|
}
|
|
for item in attempted_models
|
|
if item.get("failure_type")
|
|
]
|
|
return metadata
|
|
|
|
|
|
def _decompose_with_control_models(
|
|
*,
|
|
llm_key: str,
|
|
base_url: str,
|
|
control_models: list[str],
|
|
failover_enabled: bool,
|
|
budget_manager,
|
|
cost_ledger=None,
|
|
user_query: str,
|
|
completed_artifacts: dict | None,
|
|
original_contracts: dict[str, dict] | None,
|
|
capability_cards: list[CapabilityCard] | None = None,
|
|
sync_model_transport: SyncModelTransportPort | None = None,
|
|
response_modes: Mapping[str, str] | None = None,
|
|
planner_max_output_tokens: int | None = None,
|
|
planner_role_policy: ControlRoleInvocationPolicyV1 | None = None,
|
|
planner_attempt_evidence_path: str | None = None,
|
|
) -> tuple[SGARPlanner, PlannerOutput, list[dict]]:
|
|
attempted: list[dict] = []
|
|
last_error: Exception | None = None
|
|
for idx, model_id in enumerate(control_models):
|
|
planner = SGARPlanner(
|
|
api_key=llm_key,
|
|
base_url=base_url,
|
|
model=model_id,
|
|
budget_manager=budget_manager,
|
|
cost_ledger=cost_ledger,
|
|
transport=sync_model_transport,
|
|
strict_response_schema=True,
|
|
response_mode=(response_modes or {}).get(model_id),
|
|
max_output_tokens=planner_max_output_tokens,
|
|
role_policy=planner_role_policy,
|
|
attempt_evidence_path=planner_attempt_evidence_path,
|
|
)
|
|
try:
|
|
output = planner.decompose_task(
|
|
user_query,
|
|
completed_artifacts=completed_artifacts,
|
|
original_contracts=original_contracts,
|
|
capability_cards=capability_cards,
|
|
)
|
|
parse_mode = str(getattr(planner, "last_parse_metadata", {}).get("parse_mode") or "")
|
|
if (
|
|
failover_enabled
|
|
and parse_mode == "deterministic_fallback"
|
|
and idx < len(control_models) - 1
|
|
):
|
|
attempted.append(
|
|
{
|
|
"model_id": model_id,
|
|
"status": "failed",
|
|
"failure_type": "schema_exhausted",
|
|
"reason": "Planner exhausted provider JSON/schema modes and used deterministic fallback.",
|
|
}
|
|
)
|
|
logger.warning(
|
|
"[ControlModel] Planner model {} exhausted schema modes; trying next control model.",
|
|
model_id,
|
|
)
|
|
continue
|
|
# A deterministic DAG is a valid planner result. It must proceed
|
|
# through Router and real execution; treating it as schema
|
|
# exhaustion prevented every Tool case from reaching execution
|
|
# whenever the provider emitted a locally repairable contract.
|
|
attempted.append({"model_id": model_id, "status": "success"})
|
|
planner.last_parse_metadata = _planner_parse_metadata_with_control_trace(
|
|
planner,
|
|
selected_model=model_id,
|
|
attempted_models=attempted,
|
|
)
|
|
return planner, output, attempted
|
|
except ModelAccountingError:
|
|
raise
|
|
except (asyncio.CancelledError, KeyboardInterrupt):
|
|
raise
|
|
except Exception as exc:
|
|
last_error = exc
|
|
failure_type, message = classify_control_model_exception(exc)
|
|
attempted.append(
|
|
{
|
|
"model_id": model_id,
|
|
"status": "failed",
|
|
"failure_type": failure_type,
|
|
"reason": message,
|
|
}
|
|
)
|
|
if not failover_enabled or idx >= len(control_models) - 1 or not is_control_model_failover_failure(failure_type):
|
|
raise
|
|
logger.warning(
|
|
"[ControlModel] Planner model {} failed with {}; trying next control model.",
|
|
model_id,
|
|
failure_type,
|
|
)
|
|
raise ModelResponseContractError("control_model_chain_exhausted") from last_error
|
|
|
|
|
|
def _planner_generation_failure(
|
|
exc: BaseException,
|
|
*,
|
|
run_id: str,
|
|
) -> TerminalFailureEnvelope:
|
|
if isinstance(exc, PlannerGenerationError):
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility=exc.responsibility,
|
|
failure_stage=exc.failure_stage,
|
|
failure_code=exc.failure_code,
|
|
exception=exc,
|
|
retryable=exc.retryable,
|
|
response_received=exc.response_received,
|
|
run_id=run_id,
|
|
request_sha256=exc.request_sha256,
|
|
)
|
|
if isinstance(exc, BudgetControlError):
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility="budget",
|
|
failure_stage="planner_generation",
|
|
failure_code=str(getattr(exc, "error_code", "model_cost_limit_reached")),
|
|
exception=exc,
|
|
run_id=run_id,
|
|
)
|
|
if isinstance(exc, ModelAccountingError):
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="model_accounting",
|
|
failure_code=str(getattr(exc, "error_code", "model_accounting_failure")),
|
|
exception=exc,
|
|
run_id=run_id,
|
|
)
|
|
if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt)):
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility="interrupted",
|
|
failure_stage="planner_generation",
|
|
failure_code="planner_generation_interrupted",
|
|
exception=exc,
|
|
run_id=run_id,
|
|
)
|
|
failure_type, _message = classify_control_model_exception(
|
|
exc if isinstance(exc, Exception) else Exception(type(exc).__name__)
|
|
)
|
|
if failure_type in {
|
|
"schema_exhausted",
|
|
"json_parse_exhausted",
|
|
"empty_response_exhausted",
|
|
}:
|
|
responsibility = "research"
|
|
failure_code = "planner_generation_schema_invalid"
|
|
response_received = True
|
|
elif failure_type in {
|
|
"provider_connection_error",
|
|
"provider_stream_error",
|
|
"provider_rate_limit",
|
|
"provider_server_error",
|
|
"provider_authorization_error",
|
|
"provider_model_unavailable",
|
|
"provider_non_retryable_status",
|
|
"model_unavailable",
|
|
}:
|
|
responsibility = "infrastructure"
|
|
failure_code = failure_type
|
|
response_received = False
|
|
else:
|
|
responsibility = "framework"
|
|
failure_code = "planner_generation_control_failure"
|
|
response_received = False
|
|
return TerminalFailureEnvelope.create(
|
|
responsibility=responsibility,
|
|
failure_stage="planner_generation",
|
|
failure_code=failure_code,
|
|
exception=exc,
|
|
retryable=responsibility == "infrastructure",
|
|
response_received=response_received,
|
|
run_id=run_id,
|
|
)
|
|
|
|
|
|
async def _run_pipeline_with_cost_ledger(
|
|
config: dict,
|
|
user_query: str,
|
|
output_dir: str,
|
|
report_path: str,
|
|
cost_ledger: RunCostLedger,
|
|
execution_ledger: RunExecutionLedger,
|
|
recovery_policy: RecoveryPolicy,
|
|
recovery_ledger: RecoveryEventLedger,
|
|
evaluator_policy: EvaluatorPolicy | None,
|
|
evaluation_ledger: EvaluationEventLedger,
|
|
artifact_store: ArtifactLifecycleStore,
|
|
context_commit_store: ContextCommitStore,
|
|
system_role_probe_audit: SystemRoleProbeAuditStore,
|
|
query_diagnostics: dict | None = None,
|
|
task_invocation: PreparedTaskInvocation | None = None,
|
|
network_policy_mode: str = "disabled",
|
|
planner_variant: str = "resource_aware",
|
|
runtime_authority: str = "git",
|
|
execution_substrate: object | None = None,
|
|
execution_substrate_mode: str = "default",
|
|
require_final_delivery: bool = True,
|
|
) -> str:
|
|
llm_key = config["llm_key"]
|
|
llm_settings = config.get("llm_settings", {})
|
|
evaluation_mode = normalize_evaluation_mode(
|
|
llm_settings.get("evaluation_mode")
|
|
)
|
|
capability_probe_enforcement_policy = (
|
|
normalize_capability_probe_enforcement_policy(
|
|
llm_settings.get("capability_probe_enforcement_policy")
|
|
)
|
|
)
|
|
control_role_policy_path = _runtime_policy_path(
|
|
config,
|
|
"control_role_policy_path",
|
|
"sgar_mvp/config/control_role_policy.json",
|
|
)
|
|
retrieval_policy_path = _runtime_policy_path(
|
|
config,
|
|
"retrieval_policy_path",
|
|
"sgar_mvp/config/retrieval_policy.json",
|
|
)
|
|
model_profile_path = None
|
|
runtime_settings = config.get("runtime_settings")
|
|
if isinstance(runtime_settings, Mapping) and runtime_settings.get("model_profile_path"):
|
|
model_profile_path = _runtime_policy_path(
|
|
config,
|
|
"model_profile_path",
|
|
"sgar_mvp/config/model_profile.json",
|
|
)
|
|
# Bind the selected policy before any resource-loader path can import the
|
|
# lazy retrieval backend. This is essential for isolated experiment
|
|
# policies and is a no-op identity check for the default production policy.
|
|
bind_retrieval_policy_path(retrieval_policy_path)
|
|
control_role_policy = load_control_role_policy(control_role_policy_path)
|
|
retrieval_policy = load_retrieval_policy(retrieval_policy_path)
|
|
if model_profile_path is None and control_role_policy.protocol == CONTROL_ROLE_POLICY_PROTOCOL:
|
|
model_profile_path = Path(PROJECT_ROOT) / "sgar_mvp/config/model_profile.json"
|
|
if model_profile_path is not None:
|
|
profile = load_control_model_profile(model_profile_path)
|
|
try:
|
|
require_policy_matches_model_profile(control_role_policy, profile)
|
|
except ValueError as exc:
|
|
raise RuntimeError("control_policy_model_profile_mismatch") from exc
|
|
if evaluation_mode != profile.evaluation_mode:
|
|
raise RuntimeError("evaluation_mode_model_profile_mismatch")
|
|
if (
|
|
retrieval_policy.embedding_model != profile.embedding_model
|
|
or retrieval_policy.embedding_candidate_id != profile.embedding_candidate_id
|
|
):
|
|
raise RuntimeError("embedding_model_profile_mismatch")
|
|
base_url = llm_settings.get("base_url", "https://api.openai.com/v1")
|
|
configured_model = llm_settings.get("model", DEFAULT_SYSTEM_MODEL_CHAIN[0])
|
|
model_transport_bundle = create_production_model_transport_bundle(
|
|
api_key=llm_key,
|
|
base_url=base_url,
|
|
)
|
|
if runtime_authority not in {"git", "release"}:
|
|
raise RuntimeError("runtime_authority_invalid")
|
|
if (
|
|
runtime_authority == "release"
|
|
and control_role_policy.protocol != CONTROL_ROLE_POLICY_PROTOCOL
|
|
):
|
|
raise RuntimeError("experimental_control_policy_forbidden_in_release")
|
|
planner_policy = control_role_policy.for_role("planner")
|
|
configured_chain = tuple(llm_settings.get("system_model_chain") or ())
|
|
if configured_model != planner_policy.api_model_id or configured_chain not in {
|
|
(),
|
|
(planner_policy.api_model_id,),
|
|
}:
|
|
raise RuntimeError("configured_planner_control_policy_mismatch")
|
|
profiler_policy = control_role_policy.for_role("profiler")
|
|
if (
|
|
retrieval_policy.hyde.resource_id != profiler_policy.resource_id
|
|
or retrieval_policy.hyde.api_model_id != profiler_policy.api_model_id
|
|
or retrieval_policy.hyde.reasoning_effort != profiler_policy.reasoning_effort
|
|
or retrieval_policy.hyde.temperature != profiler_policy.temperature
|
|
or retrieval_policy.control_role_policy_sha256
|
|
!= control_role_policy.policy_sha256
|
|
):
|
|
raise RuntimeError("profiler_control_policy_mismatch")
|
|
if evaluation_mode != "off":
|
|
if evaluator_policy is None:
|
|
raise RuntimeError("evaluator_policy_missing")
|
|
evaluator_role_policy = control_role_policy.for_role("evaluator")
|
|
if (
|
|
evaluator_policy.model_resource_id != evaluator_role_policy.resource_id
|
|
or evaluator_policy.reasoning_effort != evaluator_role_policy.reasoning_effort
|
|
or evaluator_policy.temperature != evaluator_role_policy.temperature
|
|
):
|
|
raise RuntimeError("evaluator_control_policy_mismatch")
|
|
if runtime_authority == "release":
|
|
provider_probe_receipt_path, provider_probe_receipt = (
|
|
resolve_activated_release_provider_probe_receipt(
|
|
PROJECT_ROOT,
|
|
expected_endpoint_identity_sha256=(
|
|
model_transport_bundle.endpoint_identity.identity_sha256
|
|
),
|
|
)
|
|
)
|
|
system_role_probe_audit.bind_sealed_receipt(
|
|
receipt_file_sha256=hashlib.sha256(
|
|
provider_probe_receipt_path.read_bytes()
|
|
).hexdigest(),
|
|
result_sha256=str(provider_probe_receipt["result_sha256"]),
|
|
)
|
|
else:
|
|
provider_probe_receipt_path, provider_probe_receipt = load_git_control_probe_receipt(
|
|
PROJECT_ROOT, config=config,
|
|
expected_endpoint_identity_sha256=(
|
|
model_transport_bundle.endpoint_identity.identity_sha256
|
|
),
|
|
control_role_policy=control_role_policy,
|
|
)
|
|
provider_probe_records = {
|
|
str(item["role"]): dict(item) for item in provider_probe_receipt["records"]
|
|
}
|
|
if tuple(provider_probe_records) != RELEASE_PROVIDER_PROBE_ROLES:
|
|
raise RuntimeError("control_probe_role_order_mismatch")
|
|
public_query_reference = _public_query_reference(user_query)
|
|
trace_path = os.path.join(output_dir, "trace.jsonl")
|
|
if os.path.exists(trace_path):
|
|
os.remove(trace_path)
|
|
runtime_mode = "external" if execution_substrate is not None else "native"
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "runtime_mode",
|
|
"stage": "main_startup",
|
|
"runtime_mode": runtime_mode,
|
|
"execution_substrate_mode": execution_substrate_mode,
|
|
"execution_substrate_id": (
|
|
str(getattr(execution_substrate, "runtime_id", ""))
|
|
if execution_substrate is not None
|
|
else None
|
|
),
|
|
},
|
|
)
|
|
retrieval_identity = build_retrieval_runtime_identity(
|
|
project_root=PROJECT_ROOT,
|
|
provider_compatibility=_provider_compatibility_projection(base_url),
|
|
provider_endpoint_identity_sha256=(
|
|
model_transport_bundle.endpoint_identity.identity_sha256
|
|
),
|
|
require_release_sealed=(runtime_authority == "release"),
|
|
honor_release_environment=(runtime_authority == "release"),
|
|
policy_path=retrieval_policy_path,
|
|
)
|
|
_atomic_write_json(
|
|
os.path.join(output_dir, "retrieval_runtime_identity.json"),
|
|
retrieval_identity.model_dump(mode="json"),
|
|
)
|
|
applied_ready_state = None
|
|
if runtime_authority == "git":
|
|
applied_ready_state = load_applied_model_ready_state(
|
|
PROJECT_ROOT,
|
|
expected_endpoint_identity_sha256=(
|
|
model_transport_bundle.endpoint_identity.identity_sha256
|
|
),
|
|
)
|
|
system_role_probe_audit.bind_git_runtime(
|
|
applied_ready_state_sha256=applied_ready_state.health_sha256,
|
|
control_probe_receipt_sha256=hashlib.sha256(
|
|
provider_probe_receipt_path.read_bytes()
|
|
).hexdigest(),
|
|
control_probe_result_sha256=str(provider_probe_receipt["result_sha256"]),
|
|
)
|
|
capability_probe_service: Any = AppliedReadyStateCapabilityService(
|
|
applied_ready_state,
|
|
enforcement_policy=capability_probe_enforcement_policy,
|
|
)
|
|
else:
|
|
capability_probe_service = ExactCapabilityProbeService(
|
|
model_transport_bundle.sync,
|
|
cache_path=os.path.join(
|
|
PROJECT_ROOT,
|
|
".sgar_cache",
|
|
"model_response_capability_probes.json",
|
|
),
|
|
enforcement_policy=capability_probe_enforcement_policy,
|
|
)
|
|
|
|
retrieval_coordinator = RetrievalCoordinator(
|
|
retrieval_identity,
|
|
sync_model_transport=model_transport_bundle.sync,
|
|
capability_probe_service=capability_probe_service,
|
|
runtime_incompatibility=(
|
|
getattr(execution_substrate, "resource_incompatibility", None)
|
|
if execution_substrate is not None
|
|
else None
|
|
),
|
|
model_readiness_authority=(
|
|
"applied_ready_state" if runtime_authority == "git" else "live_probe"
|
|
),
|
|
)
|
|
# Fixed internal roles have independent evidence, outside candidate health.
|
|
control_selector = ControlModelSelector(
|
|
configured_chain=[control_role_policy.for_role("planner").api_model_id],
|
|
available_models=[str(provider_probe_records["planner"]["api_model_id"])],
|
|
health_gate=("sealed_role_probes" if runtime_authority == "release"
|
|
else "verified_role_probes"),
|
|
health_path=str(provider_probe_receipt_path),
|
|
health_loaded=True,
|
|
)
|
|
control_model_chain = control_selector.execution_chain(configured_model)
|
|
model = control_selector.primary_model(configured_model)
|
|
planner_role_policy = control_role_policy.for_role("planner")
|
|
if model != planner_role_policy.api_model_id:
|
|
raise RuntimeError("sealed_planner_control_policy_model_mismatch")
|
|
if not control_selector.available_models and llm_settings.get("system_model_chain"):
|
|
logger.error(
|
|
"[ControlModel] No health-gated control models available: {}",
|
|
control_selector.public_projection(project_root=PROJECT_ROOT),
|
|
)
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="infrastructure",
|
|
failure_stage="control_model_health_gate",
|
|
failure_code="control_model_health_chain_unavailable",
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return PipelineTerminalStatus(
|
|
"structured_failure",
|
|
responsibility=failure.responsibility,
|
|
failure_stage=failure.failure_stage,
|
|
failure_code=failure.failure_code,
|
|
failure=failure,
|
|
)
|
|
environment_profile = scan_environment(
|
|
probe_docker=(runtime_authority == "release")
|
|
)
|
|
# Single source of truth for the execution runtime image. Executors read the
|
|
# same env var (SGAR_RUNTIME_IMAGE); config.docker_python_image can override.
|
|
runtime_image = os.environ.get("SGAR_RUNTIME_IMAGE") or llm_settings.get(
|
|
"docker_python_image", "python:3.11"
|
|
)
|
|
os.environ["SGAR_RUNTIME_IMAGE"] = runtime_image
|
|
if runtime_authority == "release" and environment_profile.docker_available:
|
|
environment_profile.docker_warmup = warmup_docker_runtime(
|
|
environment_profile,
|
|
image=runtime_image,
|
|
timeout_sec=llm_settings.get("docker_warmup_timeout_sec", 300),
|
|
workspace_root=PROJECT_ROOT,
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "environment_profile",
|
|
"stage": "main_startup",
|
|
"profile": environment_profile.public_projection(),
|
|
},
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "control_model_chain",
|
|
"stage": "main_startup",
|
|
**control_selector.public_projection(project_root=PROJECT_ROOT),
|
|
},
|
|
)
|
|
logger.info(
|
|
"[ControlModel] configured={} available={} skipped={}",
|
|
control_selector.configured_chain,
|
|
control_selector.available_models,
|
|
control_selector.skipped_models,
|
|
)
|
|
logger.info(
|
|
"[Environment] docker_cli={} docker_daemon={} docker_ready={} reason={}",
|
|
environment_profile.docker_cli_available,
|
|
environment_profile.docker_daemon_available,
|
|
environment_profile.docker_available,
|
|
(
|
|
"ok"
|
|
if not environment_profile.docker_error
|
|
else canonical_sha256(environment_profile.docker_error)
|
|
),
|
|
)
|
|
if environment_profile.docker_warmup:
|
|
logger.info(
|
|
"[Environment] docker_warmup={}",
|
|
environment_profile.public_projection()["docker_warmup"],
|
|
)
|
|
|
|
# Existing orchestration code reads the compatibility budget properties,
|
|
# while the metered transport remains the sole charging authority.
|
|
budget_manager = cost_ledger
|
|
|
|
planner = None
|
|
completed_artifacts = {}
|
|
original_contracts: dict[str, dict] = {}
|
|
replan_history: list[dict] = []
|
|
# Experiments can disable graph-level recovery without changing the normal
|
|
# product default. A value of zero means one planner pass and no hidden
|
|
# graph replan or granularity patch.
|
|
MAX_PLANNER_REPLANS = max(0, int(llm_settings.get("max_planner_replans", 2)))
|
|
replan_count = 0
|
|
pipeline_status = "structured_failure"
|
|
|
|
# Load the exact effective pool/index once for the complete run. Its
|
|
# in-memory identity is checked against the pre-paid immutable snapshot and
|
|
# never refreshed after Planner or execution begins.
|
|
control_index = control_model_index(
|
|
load_resource_index_static().values(), root=PROJECT_ROOT
|
|
)
|
|
control_definitions = _build_resource_definitions(control_index)
|
|
library, fallback_ids, resource_index = load_real_pool_with_index()
|
|
library, fallback_ids, resource_index = _filter_models_for_provider(
|
|
library,
|
|
fallback_ids,
|
|
resource_index,
|
|
llm_settings,
|
|
base_url,
|
|
)
|
|
eligible_resource_ids = set(retrieval_identity.eligible_resource_ids)
|
|
library = [item for item in library if item.id in eligible_resource_ids]
|
|
fallback_ids = [
|
|
resource_id
|
|
for resource_id in fallback_ids
|
|
if resource_id in eligible_resource_ids
|
|
]
|
|
resource_index = {
|
|
resource_id: raw
|
|
for resource_id, raw in resource_index.items()
|
|
if resource_id in eligible_resource_ids
|
|
}
|
|
_validate_loaded_retrieval_inputs(
|
|
retrieval_identity,
|
|
library,
|
|
resource_index,
|
|
policy_path=retrieval_policy_path,
|
|
)
|
|
capability_consistency = build_capability_consistency_report(resource_index)
|
|
capability_report_path = os.path.join(
|
|
output_dir,
|
|
"plan_compiler",
|
|
"capability_catalog_consistency.json",
|
|
)
|
|
_atomic_write_json(
|
|
capability_report_path,
|
|
capability_consistency.model_dump(mode="json"),
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "capability_catalog_consistency",
|
|
"stage": "main_startup",
|
|
"resource_count": capability_consistency.resource_count,
|
|
"sealable_resource_count": (
|
|
capability_consistency.sealable_resource_count
|
|
),
|
|
"report_sha256": capability_consistency.report_sha256,
|
|
},
|
|
)
|
|
if (
|
|
capability_consistency.sealable_resource_count
|
|
!= capability_consistency.resource_count
|
|
):
|
|
raise RuntimeError("resource_capability_catalog_unsealable")
|
|
planner_probe_record = provider_probe_records["planner"]
|
|
model = str(planner_probe_record["api_model_id"])
|
|
control_resource_id = _resource_id_for_api_model(control_index, model)
|
|
if control_resource_id != planner_probe_record["model_resource_id"]:
|
|
raise RuntimeError("control_planner_probe_resource_identity_mismatch")
|
|
control_model_chain = [model]
|
|
verified_control_modes = {model: "native_strict_schema"}
|
|
system_role_probe_audit.select_model(
|
|
role="planner",
|
|
resource_id=control_resource_id,
|
|
model_id=model,
|
|
response_mode="native_strict_schema",
|
|
)
|
|
profiler_probe_record = provider_probe_records["profiler"]
|
|
if (
|
|
profiler_probe_record["model_resource_id"]
|
|
!= retrieval_identity.hyde_resource_id
|
|
or profiler_probe_record["api_model_id"]
|
|
!= retrieval_identity.hyde_api_model_id
|
|
):
|
|
raise RuntimeError("control_profiler_probe_resource_identity_mismatch")
|
|
retrieval_coordinator.set_hyde_response_mode("native_strict_schema")
|
|
system_role_probe_audit.select_model(
|
|
role="profiler",
|
|
resource_id=retrieval_identity.hyde_resource_id,
|
|
model_id=retrieval_identity.hyde_api_model_id,
|
|
response_mode="native_strict_schema",
|
|
)
|
|
planner_capability_cards: list[CapabilityCard] = []
|
|
if planner_variant == "resource_aware":
|
|
planner_capability_cards = list(
|
|
build_capability_cards(resource_index).values()
|
|
)
|
|
if not planner_capability_cards:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="planner_startup",
|
|
failure_code="planner_capability_context_empty",
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
planner_capability_context_sha256 = canonical_sha256(
|
|
[card.model_dump(mode="json") for card in planner_capability_cards]
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "planner_capability_context",
|
|
"stage": "planner_startup",
|
|
"planner_variant": planner_variant,
|
|
"card_count": len(planner_capability_cards),
|
|
"capability_context_sha256": planner_capability_context_sha256,
|
|
"candidate_pool_effect": "abstract_role_context_only",
|
|
},
|
|
)
|
|
try:
|
|
resource_definitions = _build_resource_definitions(resource_index)
|
|
except RetrievalRuntimeError as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="resource_runtime_startup",
|
|
failure_code="resource_definition_index_identity_mismatch",
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
evaluator_manifest = None
|
|
evaluator_definition = None
|
|
resolved_evaluator_model = None
|
|
if evaluation_mode != "off":
|
|
assert evaluator_policy is not None
|
|
evaluator_manifest = control_index.get(evaluator_policy.model_resource_id)
|
|
if not isinstance(evaluator_manifest, dict):
|
|
raise EvaluationRuntimeError("evaluator_model_not_in_control_catalog")
|
|
evaluator_definition = control_definitions.get(evaluator_policy.model_resource_id)
|
|
if evaluator_definition is None:
|
|
raise EvaluationRuntimeError("evaluator_resource_definition_missing")
|
|
evaluator_response_mode = _plan_compiler_response_mode(
|
|
evaluator_manifest,
|
|
configured_mode=llm_settings.get("evaluator_response_mode"),
|
|
)
|
|
resolved_evaluator_model = resolve_evaluator_model(
|
|
policy=evaluator_policy,
|
|
pricing_catalog=cost_ledger.catalog,
|
|
manifest=evaluator_manifest,
|
|
availability_status=evaluator_definition.status,
|
|
response_mode=evaluator_response_mode,
|
|
)
|
|
strict_plan_only = recovery_policy.sealed_runtime_mode == "strict_plan_only"
|
|
full_generation_manifest = None
|
|
full_generation_definition = None
|
|
full_generation_response_mode = None
|
|
system_full_generation_policy = None
|
|
if not strict_plan_only:
|
|
full_generation_manifest = control_index.get(
|
|
recovery_policy.full_generation_model_resource_id
|
|
)
|
|
if not isinstance(full_generation_manifest, dict):
|
|
raise RecoveryControlError("full_generation_model_not_in_control_catalog")
|
|
full_generation_definition = control_definitions.get(
|
|
recovery_policy.full_generation_model_resource_id
|
|
)
|
|
if full_generation_definition is None:
|
|
raise RecoveryControlError("full_generation_resource_definition_missing")
|
|
full_generation_response_mode = _plan_compiler_response_mode(
|
|
full_generation_manifest,
|
|
configured_mode=llm_settings.get("full_generation_response_mode"),
|
|
)
|
|
system_full_generation_policy = resolve_system_full_generation_policy(
|
|
recovery_policy=recovery_policy,
|
|
pricing_catalog=cost_ledger.catalog,
|
|
manifest=full_generation_manifest,
|
|
response_mode=full_generation_response_mode,
|
|
temperature=float(llm_settings.get("execution_temperature", 0.5)),
|
|
max_tokens=int(llm_settings.get("execution_max_tokens", 8192)),
|
|
allow_streaming=bool(llm_settings.get("execution_allow_streaming", True)),
|
|
prompt_version=FULL_GENERATION_PROMPT_VERSION,
|
|
prompt_sha256=FULL_GENERATION_PROMPT_SHA256,
|
|
availability_status=full_generation_definition.status,
|
|
)
|
|
gen_models = [
|
|
manifest.model_dump()
|
|
for manifest in library
|
|
if manifest.type == ManifestType.MODEL and manifest.id in fallback_ids
|
|
]
|
|
router = SGARRouter(
|
|
confidence_threshold=0.75,
|
|
# Retrieval confidence is evidence-only in formal runtime v1.
|
|
granularity_threshold=0.0,
|
|
bundle_advantage_threshold=llm_settings.get("bundle_advantage_threshold", 1.0),
|
|
# The formal Router is a deterministic control plane. Keeping the
|
|
# legacy policy transport absent makes accidental model-policy entry
|
|
# fail before any provider request.
|
|
policy_api_key="",
|
|
policy_base_url=base_url,
|
|
policy_model=llm_settings.get("router_policy_model", model),
|
|
policy_model_chain=control_model_chain,
|
|
policy_response_modes={},
|
|
baseline_model_id=llm_settings.get("full_generation_baseline_model", model),
|
|
resource_index=resource_index,
|
|
execute_low_advantage_on_exhaustion=llm_settings.get(
|
|
"execute_low_advantage_on_exhaustion", True
|
|
),
|
|
# Formal frozen sessions never enter legacy candidate completion. Keep
|
|
# the constructor flag false as an additional static/runtime barrier.
|
|
enable_intelligent_resource_completion=False,
|
|
supplemental_model_ids=[],
|
|
max_supplemental_models=0,
|
|
cost_ledger=cost_ledger,
|
|
policy_transport=None,
|
|
)
|
|
compiler_invocation_policy = control_role_policy.for_role("plan_compiler")
|
|
adaptation_invocation_policy = control_role_policy.for_role("plan_adaptation")
|
|
compiler_target_price = cost_ledger.catalog.resolve(
|
|
model_ref=compiler_invocation_policy.resource_id
|
|
)
|
|
if (
|
|
compiler_target_price.resource_id != compiler_invocation_policy.resource_id
|
|
or compiler_target_price.api_model_id != compiler_invocation_policy.api_model_id
|
|
):
|
|
raise ModelResponseContractError("control_role_policy_catalog_identity_mismatch")
|
|
compiler_target_manifest = control_index.get(compiler_target_price.resource_id)
|
|
if not isinstance(compiler_target_manifest, dict):
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="plan_compiler_startup",
|
|
failure_code="plan_compiler_model_not_in_control_catalog",
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
compiler_targets: list[dict[str, Any]] = [
|
|
{
|
|
"resource_id": compiler_target_price.resource_id,
|
|
"api_model_id": compiler_target_price.api_model_id,
|
|
"manifest": compiler_target_manifest,
|
|
}
|
|
]
|
|
required_role_models = []
|
|
if resolved_evaluator_model is not None:
|
|
required_role_models.append(
|
|
(
|
|
"evaluator",
|
|
resolved_evaluator_model.resource_id,
|
|
resolved_evaluator_model.api_model_id,
|
|
)
|
|
)
|
|
if system_full_generation_policy is not None:
|
|
raise RecoveryControlError("sealed_release_forbids_full_generation")
|
|
for compiler_target in compiler_targets:
|
|
required_role_models.extend(
|
|
(
|
|
(
|
|
"plan_compiler",
|
|
compiler_target["resource_id"],
|
|
compiler_target["api_model_id"],
|
|
),
|
|
(
|
|
"plan_adaptation",
|
|
compiler_target["resource_id"],
|
|
compiler_target["api_model_id"],
|
|
),
|
|
)
|
|
)
|
|
verified_role_modes: dict[str, str] = {}
|
|
verified_role_identity_modes: dict[tuple[str, str, str], str] = {}
|
|
for role, resource_id, api_model_id in required_role_models:
|
|
expected_policy = control_role_policy.for_role(role)
|
|
record = provider_probe_records[role]
|
|
if (
|
|
record["model_resource_id"] != resource_id
|
|
or record["api_model_id"] != api_model_id
|
|
):
|
|
raise RuntimeError(f"control_{role}_probe_resource_identity_mismatch")
|
|
if record["reasoning_effort"] != expected_policy.reasoning_effort:
|
|
raise RuntimeError(f"control_{role}_probe_reasoning_effort_mismatch")
|
|
if record.get("temperature") != expected_policy.temperature:
|
|
raise RuntimeError(f"control_{role}_probe_temperature_mismatch")
|
|
verified_role_modes.setdefault(role, "native_strict_schema")
|
|
verified_role_identity_modes[(role, resource_id, api_model_id)] = (
|
|
"native_strict_schema"
|
|
)
|
|
system_role_probe_audit.select_model(
|
|
role=role,
|
|
resource_id=resource_id,
|
|
model_id=api_model_id,
|
|
response_mode="native_strict_schema",
|
|
)
|
|
if evaluation_mode != "off":
|
|
assert evaluator_policy is not None
|
|
assert evaluator_manifest is not None
|
|
assert evaluator_definition is not None
|
|
evaluator_response_mode = verified_role_modes["evaluator"]
|
|
resolved_evaluator_model = resolve_evaluator_model(
|
|
policy=evaluator_policy,
|
|
pricing_catalog=cost_ledger.catalog,
|
|
manifest=evaluator_manifest,
|
|
availability_status=evaluator_definition.status,
|
|
response_mode=evaluator_response_mode,
|
|
)
|
|
primary_compiler_target = compiler_targets[0]
|
|
plan_response_mode = verified_role_identity_modes[
|
|
(
|
|
"plan_compiler",
|
|
primary_compiler_target["resource_id"],
|
|
primary_compiler_target["api_model_id"],
|
|
)
|
|
]
|
|
system_role_probe_audit.mark_verified()
|
|
compiler_store = PlanCompilationStore(
|
|
Path(output_dir),
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
executable_plan_compiler = ExecutablePlanCompiler(
|
|
transport=model_transport_bundle.sync,
|
|
cost_ledger=cost_ledger,
|
|
compiler_model_resource_id=primary_compiler_target["resource_id"],
|
|
compiler_model_api_id=primary_compiler_target["api_model_id"],
|
|
store=compiler_store,
|
|
response_mode=verified_role_identity_modes[
|
|
(
|
|
"plan_compiler",
|
|
primary_compiler_target["resource_id"],
|
|
primary_compiler_target["api_model_id"],
|
|
)
|
|
],
|
|
adaptation_response_mode=verified_role_identity_modes[
|
|
(
|
|
"plan_adaptation",
|
|
primary_compiler_target["resource_id"],
|
|
primary_compiler_target["api_model_id"],
|
|
)
|
|
],
|
|
control_role_policy=control_role_policy,
|
|
capability_probe_service=capability_probe_service,
|
|
candidate_compact_quotas=retrieval_policy.compact_quotas(),
|
|
candidate_compact_total_max=retrieval_policy.compact_total_max,
|
|
)
|
|
full_generation_executor = (
|
|
FullGenerationExecutor(
|
|
transport=model_transport_bundle.sync,
|
|
cost_ledger=cost_ledger,
|
|
system_policy=system_full_generation_policy,
|
|
)
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
)
|
|
evaluation_coordinator = (
|
|
EvaluationCoordinator(
|
|
transport=model_transport_bundle.async_port,
|
|
resolved_model=resolved_evaluator_model,
|
|
policy=evaluator_policy,
|
|
cost_ledger=cost_ledger,
|
|
event_ledger=evaluation_ledger,
|
|
)
|
|
if evaluation_mode != "off"
|
|
else StaticEvaluationCoordinator(
|
|
mode="off",
|
|
event_ledger=evaluation_ledger,
|
|
)
|
|
)
|
|
static_evaluation_coordinator = StaticEvaluationCoordinator(
|
|
mode="off",
|
|
event_ledger=evaluation_ledger,
|
|
)
|
|
artifact_lifecycle_coordinator = ArtifactLifecycleCoordinator(
|
|
artifact_store=artifact_store,
|
|
context_store=context_commit_store,
|
|
evaluation_coordinator=evaluation_coordinator,
|
|
evaluation_mode=evaluation_mode,
|
|
static_evaluation_coordinator=static_evaluation_coordinator,
|
|
)
|
|
plan_runtime_capabilities = _plan_runtime_capabilities(
|
|
resource_index,
|
|
runtime_image=runtime_image,
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "plan_compiler_runtime_identity",
|
|
"stage": "main_startup",
|
|
"compiler_model_resource_id": primary_compiler_target["resource_id"],
|
|
"compiler_model_api_id": primary_compiler_target["api_model_id"],
|
|
"response_mode": plan_response_mode,
|
|
"control_role_policy_sha256": control_role_policy.policy_sha256,
|
|
"compiler_role_policy_sha256": (
|
|
compiler_invocation_policy.role_policy_sha256
|
|
),
|
|
"adaptation_role_policy_sha256": (
|
|
adaptation_invocation_policy.role_policy_sha256
|
|
),
|
|
"reasoning_effort": compiler_invocation_policy.reasoning_effort,
|
|
"temperature": compiler_invocation_policy.temperature,
|
|
"allow_model_failover": False,
|
|
"verified_compiler_model_chain": [
|
|
{
|
|
"resource_id": item["resource_id"],
|
|
"api_model_id": item["api_model_id"],
|
|
"plan_compiler_response_mode": verified_role_identity_modes[
|
|
("plan_compiler", item["resource_id"], item["api_model_id"])
|
|
],
|
|
"plan_adaptation_response_mode": verified_role_identity_modes[
|
|
("plan_adaptation", item["resource_id"], item["api_model_id"])
|
|
],
|
|
}
|
|
for item in compiler_targets
|
|
],
|
|
"pricing_catalog_sha256": cost_ledger.catalog.pricing_catalog_sha256,
|
|
"runtime_capabilities_sha256": (
|
|
plan_runtime_capabilities.capabilities_sha256
|
|
),
|
|
"recovery_policy_sha256": recovery_policy.policy_sha256,
|
|
"sealed_runtime_mode": recovery_policy.sealed_runtime_mode,
|
|
"full_generation_policy_sha256": (
|
|
system_full_generation_policy.policy_sha256
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
"full_generation_model_resource_id": (
|
|
system_full_generation_policy.resource_id
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
"full_generation_model_api_id": (
|
|
system_full_generation_policy.api_model_id
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
"evaluation_mode": evaluation_mode,
|
|
"evaluator_policy_sha256": (
|
|
evaluator_policy.policy_sha256 if evaluator_policy is not None else None
|
|
),
|
|
"evaluator_model_resource_id": (
|
|
resolved_evaluator_model.resource_id if resolved_evaluator_model else None
|
|
),
|
|
"evaluator_model_api_id": (
|
|
resolved_evaluator_model.api_model_id if resolved_evaluator_model else None
|
|
),
|
|
"evaluator_response_mode": (
|
|
resolved_evaluator_model.response_mode if resolved_evaluator_model else None
|
|
),
|
|
"evaluator_identity_sha256": (
|
|
resolved_evaluator_model.identity_sha256 if resolved_evaluator_model else None
|
|
),
|
|
},
|
|
)
|
|
|
|
while replan_count <= MAX_PLANNER_REPLANS:
|
|
if replan_count > 0:
|
|
logger.warning(f"[Safety Hook] Initiating Tier-3 Graph-Level Replan ({replan_count}/{MAX_PLANNER_REPLANS})...")
|
|
|
|
# ── Phase 1: Planner ───────────────────────
|
|
logger.info("-" * 60)
|
|
logger.info("Phase 1: Task Decomposition (Planner)")
|
|
logger.info("-" * 60)
|
|
|
|
planner_query = (
|
|
task_invocation.invocation.planner_request_text()
|
|
if task_invocation is not None
|
|
else user_query
|
|
)
|
|
try:
|
|
planner, planner_output, planner_control_attempts = (
|
|
_decompose_with_control_models(
|
|
llm_key=llm_key,
|
|
base_url=base_url,
|
|
control_models=control_model_chain,
|
|
failover_enabled=False,
|
|
budget_manager=budget_manager,
|
|
cost_ledger=cost_ledger,
|
|
user_query=planner_query,
|
|
completed_artifacts=(
|
|
completed_artifacts if replan_count > 0 else None
|
|
),
|
|
original_contracts=(
|
|
original_contracts if replan_count > 0 else None
|
|
),
|
|
capability_cards=planner_capability_cards or None,
|
|
sync_model_transport=model_transport_bundle.sync,
|
|
response_modes=verified_control_modes,
|
|
# The formal Planner owns its sealed 32K -> 64K-on-length
|
|
# protocol. A legacy local config value must not widen or
|
|
# otherwise alter the first production request.
|
|
planner_max_output_tokens=DEFAULT_PLANNER_MAX_OUTPUT_TOKENS,
|
|
planner_role_policy=planner_role_policy,
|
|
planner_attempt_evidence_path=os.path.join(
|
|
output_dir,
|
|
"planner_attempts.jsonl",
|
|
),
|
|
)
|
|
)
|
|
except (asyncio.CancelledError, KeyboardInterrupt):
|
|
raise
|
|
except Exception as exc:
|
|
failure = _planner_generation_failure(
|
|
exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
if replan_count > 0:
|
|
planner_output = preserve_replan_contracts(
|
|
planner_output,
|
|
original_contracts,
|
|
completed_artifacts,
|
|
)
|
|
planner_output = _bind_authoritative_final_schema(
|
|
planner_output,
|
|
task_invocation,
|
|
)
|
|
planner_output = _bind_task_text_authority(
|
|
planner_output,
|
|
task_invocation,
|
|
)
|
|
if replan_count == 0:
|
|
original_contracts = _collect_protected_contracts(planner_output)
|
|
replan_history.append(
|
|
{
|
|
"replan_count": replan_count,
|
|
"planner_parse_metadata": getattr(planner, "last_parse_metadata", {}),
|
|
"planner_contract_audit": getattr(planner, "last_contract_audit", {}),
|
|
"planner_control_model_attempts": planner_control_attempts,
|
|
"planner_variant": planner_variant,
|
|
"planner_capability_context_sha256": planner_capability_context_sha256,
|
|
"planner_capability_resource_pool_sha256": (
|
|
planner_capability_context_sha256
|
|
),
|
|
"subtasks": [
|
|
{
|
|
"id": subtask.id,
|
|
"artifact_type": subtask.artifact_type.value,
|
|
"output_extension": subtask.output_extension,
|
|
"task_stage": subtask.task_stage.value if subtask.task_stage is not None else None,
|
|
"planning_execution_mode": (
|
|
subtask.planning_execution_mode.value
|
|
if subtask.planning_execution_mode is not None
|
|
else None
|
|
),
|
|
"capability_evidence": list(subtask.capability_evidence),
|
|
"capability_gap": subtask.capability_gap,
|
|
"depends_on": list(subtask.depends_on),
|
|
}
|
|
for subtask in planner_output.subtasks
|
|
],
|
|
"dag_edge_contract_sha256s": [
|
|
edge.edge_contract_sha256
|
|
for edge in planner_output.edge_contracts
|
|
],
|
|
"dag_edge_contracts_sha256": canonical_sha256(
|
|
[
|
|
edge.model_dump(mode="json")
|
|
for edge in planner_output.edge_contracts
|
|
]
|
|
),
|
|
"protected_contracts": list(original_contracts.keys()),
|
|
}
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "planner_trace",
|
|
"replan_count": replan_count,
|
|
"query_sha256": canonical_sha256(user_query),
|
|
"query_utf8_bytes": len(user_query.encode("utf-8")),
|
|
"planner_parse_metadata": getattr(planner, "last_parse_metadata", {}),
|
|
"planner_contract_audit": getattr(planner, "last_contract_audit", {}),
|
|
"planner_control_model_attempts": planner_control_attempts,
|
|
"subtasks": [subtask.model_dump(mode="json") for subtask in planner_output.subtasks],
|
|
"dag_edge_contracts": [
|
|
edge.model_dump(mode="json")
|
|
for edge in planner_output.edge_contracts
|
|
],
|
|
},
|
|
)
|
|
|
|
|
|
# ── Phase 2: Router Adjudication ───────────
|
|
logger.info("-" * 60)
|
|
logger.info("Phase 2: Router Adjudication")
|
|
logger.info("-" * 60)
|
|
|
|
while True:
|
|
decisions: list[RoutingDecision] = []
|
|
task_list: list[dict] = []
|
|
routing_bundles: dict[str, dict] = {
|
|
"_original_query": user_query,
|
|
"_task_invocation": (
|
|
task_invocation.invocation.model_dump(mode="json")
|
|
if task_invocation is not None
|
|
else None
|
|
),
|
|
"_query_diagnostics": query_diagnostics or {},
|
|
"_replan_history": list(replan_history),
|
|
"_environment_profile": environment_profile.public_projection(),
|
|
"_control_model_chain": control_selector.public_projection(
|
|
project_root=PROJECT_ROOT
|
|
),
|
|
"_planner_parse_metadata": getattr(planner, "last_parse_metadata", {}),
|
|
"_planner_parse_history": list(getattr(planner, "parse_metadata_history", [])),
|
|
"_model_cost_summary": cost_ledger.summary(),
|
|
"_execution_summary": execution_ledger.summary(),
|
|
"_recovery_summary": recovery_ledger.summary(),
|
|
"_evaluation_summary": evaluation_ledger.summary(),
|
|
"_artifact_summary": artifact_store.ledger.artifact_summary(),
|
|
"_context_summary": artifact_store.ledger.context_summary(),
|
|
"_retrieval_runtime_identity": retrieval_identity.model_dump(mode="json"),
|
|
"_recovery_runtime_identity": {
|
|
"recovery_policy_sha256": recovery_policy.policy_sha256,
|
|
"sealed_runtime_mode": recovery_policy.sealed_runtime_mode,
|
|
"full_generation_policy_sha256": (
|
|
system_full_generation_policy.policy_sha256
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
"full_generation_model_resource_id": (
|
|
system_full_generation_policy.resource_id
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
"full_generation_model_api_id": (
|
|
system_full_generation_policy.api_model_id
|
|
if system_full_generation_policy is not None
|
|
else None
|
|
),
|
|
},
|
|
"_dag_edge_contracts": [
|
|
edge.model_dump(mode="json")
|
|
for edge in planner_output.edge_contracts
|
|
],
|
|
"_dag_edge_contracts_sha256": canonical_sha256(
|
|
[
|
|
edge.model_dump(mode="json")
|
|
for edge in planner_output.edge_contracts
|
|
]
|
|
),
|
|
}
|
|
|
|
for subtask in planner_output.subtasks:
|
|
revision = SubtaskRevisionRef(
|
|
graph_revision=replan_count,
|
|
subtask_id=subtask.id,
|
|
subtask_revision=0,
|
|
)
|
|
public_context = _portable_public_context_for_subtask(
|
|
subtask,
|
|
task_invocation,
|
|
)
|
|
contract_projection = project_retrieval_contract(
|
|
revision,
|
|
subtask,
|
|
public_context=public_context,
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "retrieval_started",
|
|
"stage": "candidate_pool_preparation",
|
|
"revision": revision.model_dump(mode="json"),
|
|
"contract_sha256": contract_projection.contract_sha256,
|
|
"retrieval_runtime_identity_sha256": (
|
|
retrieval_identity.identity_sha256
|
|
),
|
|
},
|
|
)
|
|
try:
|
|
frozen_result = retrieval_coordinator.prepare_candidate_pool(
|
|
revision,
|
|
subtask,
|
|
retrieval_identity,
|
|
library,
|
|
resource_index,
|
|
cost_ledger,
|
|
public_context=public_context,
|
|
)
|
|
except RetrievalPreparationError as exc:
|
|
for attempt in exc.attempts:
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "retrieval_attempt",
|
|
"stage": "candidate_pool_preparation",
|
|
"revision": revision.model_dump(mode="json"),
|
|
"attempt": attempt.model_dump(mode="json"),
|
|
},
|
|
)
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "retrieval_terminal",
|
|
"stage": "candidate_pool_preparation",
|
|
"revision": revision.model_dump(mode="json"),
|
|
"contract_sha256": contract_projection.contract_sha256,
|
|
"failure_code": exc.error_code,
|
|
"failure_responsibility": exc.failure_responsibility,
|
|
"response_received": exc.response_received,
|
|
"exception_type": exc.exception_type,
|
|
"message_sha256": exc.message_sha256,
|
|
},
|
|
)
|
|
routing_bundles[subtask.id] = {
|
|
"retrieval_terminal_failure": {
|
|
"failure_code": exc.error_code,
|
|
"failure_responsibility": exc.failure_responsibility,
|
|
"response_received": exc.response_received,
|
|
"exception_type": exc.exception_type,
|
|
"message_sha256": exc.message_sha256,
|
|
}
|
|
}
|
|
routing_bundles["_model_cost_summary"] = cost_ledger.summary()
|
|
generate_report(
|
|
public_query_reference,
|
|
planner_output,
|
|
decisions,
|
|
output_path=report_path,
|
|
routing_bundles=routing_bundles,
|
|
)
|
|
logger.error(
|
|
"[Retrieval] Terminal candidate-pool failure for {}: code={} responsibility={}",
|
|
subtask.id,
|
|
exc.error_code,
|
|
exc.failure_responsibility,
|
|
)
|
|
return _structured_pipeline_failure(
|
|
responsibility=exc.failure_responsibility,
|
|
failure_stage="retrieval",
|
|
failure_code=exc.error_code,
|
|
exception_type=exc.exception_type,
|
|
message_sha256=exc.message_sha256,
|
|
response_received=exc.response_received,
|
|
)
|
|
|
|
candidate_pool_artifact = _record_frozen_retrieval(
|
|
trace_path=trace_path,
|
|
output_dir=output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
frozen_result=frozen_result,
|
|
)
|
|
session = router.start_frozen_session(subtask, frozen_result, library)
|
|
|
|
# Preserve the complete admitted node, including optional semantics.
|
|
task_list.append({
|
|
**subtask.model_dump(mode="json"),
|
|
"original_query": user_query,
|
|
})
|
|
|
|
# Compatibility-only report projection. This is one candidate,
|
|
# not the resource selected by the Plan Compiler.
|
|
top_ref = (
|
|
session.top_k_resources[0]
|
|
if session.top_k_resources
|
|
else session.frozen_candidate_resources[0]
|
|
)
|
|
top_manifest = next(m for m in library if m.id == top_ref.resource_id)
|
|
legacy_decision = RoutingDecision(
|
|
mode=ExecutionMode.SEMI_GENERATIVE,
|
|
resource=top_manifest,
|
|
metrics=RoutingMetrics(
|
|
similarity=session.top_k_scores.get(top_ref.resource_id, 0.0),
|
|
advantage=top_manifest.advantage_score,
|
|
),
|
|
)
|
|
decisions.append(legacy_decision)
|
|
routing_bundles[subtask.id] = {
|
|
"execution_mode": ExecutionMode.SEMI_GENERATIVE.value,
|
|
"kwargs": {"model": model, "temperature": 0.5},
|
|
"fallback_models": gen_models,
|
|
"routing_session": session,
|
|
"router_runtime": router,
|
|
"library": library,
|
|
"resource_index": resource_index,
|
|
"original_query": user_query,
|
|
"frozen_candidate_pool": frozen_result,
|
|
"retrieval_coordinator": retrieval_coordinator,
|
|
"executable_plan_compiler": executable_plan_compiler,
|
|
"resource_definitions": {
|
|
candidate.resource_id: resource_definitions[
|
|
candidate.resource_id
|
|
]
|
|
for candidate in frozen_result.candidate_pool_snapshot.candidates
|
|
},
|
|
"model_pricing_catalog": cost_ledger.catalog,
|
|
"plan_runtime_capabilities": plan_runtime_capabilities,
|
|
"recovery_policy": recovery_policy,
|
|
"recovery_ledger": recovery_ledger,
|
|
"full_generation_executor": full_generation_executor,
|
|
"candidate_pool_artifact": candidate_pool_artifact,
|
|
}
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "routing_trace",
|
|
"subtask_id": subtask.id,
|
|
"revision": revision.model_dump(mode="json"),
|
|
"candidate_pool_sha256": (
|
|
frozen_result.candidate_pool_snapshot.candidate_pool_sha256
|
|
),
|
|
"session": session.model_dump(mode="json"),
|
|
},
|
|
)
|
|
|
|
break
|
|
|
|
routing_bundles["_model_cost_summary"] = cost_ledger.summary()
|
|
routing_bundles["_execution_summary"] = execution_ledger.summary()
|
|
routing_bundles["_recovery_summary"] = recovery_ledger.summary()
|
|
|
|
# ── Generate Visual Report ─────────────────
|
|
generate_report(
|
|
public_query_reference,
|
|
planner_output,
|
|
decisions,
|
|
output_path=report_path,
|
|
routing_bundles=routing_bundles,
|
|
)
|
|
|
|
# ── Phase 3: DAG Execution ─────────────────
|
|
logger.info("-" * 60)
|
|
logger.info("Phase 3: DAG Execution")
|
|
logger.info("-" * 60)
|
|
|
|
orchestrator = DAGOrchestrator(
|
|
llm_api_key=llm_key,
|
|
llm_base_url=base_url,
|
|
model=model,
|
|
budget_manager=budget_manager,
|
|
cost_ledger=cost_ledger,
|
|
execution_ledger=execution_ledger,
|
|
artifact_dir=output_dir,
|
|
max_same_bundle_repair_attempts=llm_settings.get(
|
|
"max_same_bundle_repair_attempts", 0
|
|
),
|
|
allow_plan_recovery=llm_settings.get("allow_plan_recovery", False),
|
|
# Stage 4A performs one evaluator handoff only. Evaluator retry and
|
|
# inconclusive review belong to Stage 4B.
|
|
max_eval_retries=0,
|
|
eval_profile_threshold_chars=llm_settings.get(
|
|
"eval_profile_threshold_chars", 12000
|
|
),
|
|
execution_strictness=llm_settings.get("execution_strictness", "balanced"),
|
|
trace_path=trace_path,
|
|
control_model_chain=control_model_chain,
|
|
control_model_metadata=control_selector.public_projection(
|
|
project_root=PROJECT_ROOT
|
|
),
|
|
environment_profile=environment_profile,
|
|
execution_max_retries=llm_settings.get("execution_max_retries", 3),
|
|
execution_max_tokens=llm_settings.get("execution_max_tokens", 8192),
|
|
execution_allow_streaming=llm_settings.get("execution_allow_streaming", True),
|
|
runtime_preparation_enabled=llm_settings.get(
|
|
"runtime_preparation_enabled", False
|
|
),
|
|
runtime_preparation_trace_path=llm_settings.get(
|
|
"runtime_preparation_trace_path"
|
|
),
|
|
runtime_preparation_run_id=llm_settings.get(
|
|
"runtime_preparation_run_id"
|
|
),
|
|
artifact_lifecycle_coordinator=artifact_lifecycle_coordinator,
|
|
artifact_store=artifact_store,
|
|
context_commit_store=context_commit_store,
|
|
# External benchmark substrates own the task filesystem. Never
|
|
# infer host files from free-form benchmark instructions there;
|
|
# task-state paths must remain runtime (/app) paths and cross the
|
|
# RPC boundary only through typed bindings/handles.
|
|
explicit_input_only=(
|
|
task_invocation is not None or execution_substrate is not None
|
|
),
|
|
network_policy_mode=network_policy_mode,
|
|
task_invocation=(
|
|
task_invocation.invocation if task_invocation is not None else None
|
|
),
|
|
async_model_transport=model_transport_bundle.async_port,
|
|
execution_substrate=execution_substrate,
|
|
execution_substrate_mode=execution_substrate_mode,
|
|
evaluation_mode=evaluation_mode,
|
|
)
|
|
if task_invocation is not None:
|
|
orchestrator.register_input_handles(task_invocation.internal_handles())
|
|
# Inject recovered contextual artifacts before executing
|
|
for k, v in completed_artifacts.items():
|
|
orchestrator.context.add_result(k, v)
|
|
|
|
try:
|
|
final_ctx = await orchestrator.run_pipeline(task_list, routing_bundles)
|
|
for task_id, bundle in routing_bundles.items():
|
|
if not isinstance(bundle, dict) or bundle.get("routing_session") is None:
|
|
continue
|
|
_append_jsonl(
|
|
trace_path,
|
|
{
|
|
"event_type": "routing_trace",
|
|
"stage": "final",
|
|
"subtask_id": task_id,
|
|
"session": bundle["routing_session"].model_dump(mode="json"),
|
|
},
|
|
)
|
|
|
|
routing_bundles["_model_cost_summary"] = cost_ledger.summary()
|
|
routing_bundles["_execution_summary"] = execution_ledger.summary()
|
|
routing_bundles["_recovery_summary"] = recovery_ledger.summary()
|
|
routing_bundles["_evaluation_summary"] = evaluation_ledger.summary()
|
|
routing_bundles["_artifact_summary"] = artifact_store.ledger.artifact_summary()
|
|
routing_bundles["_context_summary"] = artifact_store.ledger.context_summary()
|
|
|
|
# ── Phase 4: Delivery ──────────────────
|
|
logger.info("-" * 60)
|
|
logger.info("Phase 4: Final Delivery")
|
|
logger.info("-" * 60)
|
|
|
|
try:
|
|
delivery = extract_deliverables(
|
|
final_ctx,
|
|
task_list,
|
|
output_dir=output_dir,
|
|
final_deliverable_contract=(
|
|
task_invocation.invocation.final_deliverable_contract
|
|
if task_invocation is not None
|
|
else None
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
raise NodeUnrecoverableError(
|
|
TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="delivery",
|
|
failure_code="delivery_publication_failed",
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
) from exc
|
|
if delivery is not None:
|
|
publication = delivery.publication
|
|
logger.success(
|
|
" final_deliverable → {} ({} bytes, sha256={})",
|
|
publication.logical_locator,
|
|
publication.byte_size,
|
|
publication.content_sha256,
|
|
)
|
|
logger.success(
|
|
" delivery_manifest → {}",
|
|
publication.delivery_manifest_locator,
|
|
)
|
|
|
|
if task_list and delivery is None and require_final_delivery:
|
|
raise NodeUnrecoverableError(
|
|
TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="delivery",
|
|
failure_code="final_delivery_requires_committed_final_artifact",
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
)
|
|
|
|
generate_report(
|
|
public_query_reference,
|
|
planner_output,
|
|
decisions,
|
|
output_path=report_path,
|
|
routing_bundles=routing_bundles,
|
|
)
|
|
|
|
# Entire pipeline success, break out of replan loop
|
|
pipeline_status = _successful_pipeline_status(routing_bundles)
|
|
break
|
|
|
|
except NodeUnrecoverableError as e:
|
|
logger.error(
|
|
"[Pipeline] Structured node failure: {}",
|
|
type(e).__name__,
|
|
)
|
|
generate_report(
|
|
public_query_reference,
|
|
planner_output,
|
|
decisions,
|
|
output_path=report_path,
|
|
routing_bundles=routing_bundles,
|
|
)
|
|
# Preserve diagnostics, but never turn a post-freeze execution
|
|
# failure into another retrieval/Planner pass. The candidate pool
|
|
# is immutable for this Subtask revision.
|
|
completed_artifacts.update(orchestrator.context.artifacts)
|
|
logger.error(
|
|
"[Candidate Freeze] Execution failed after candidate freeze; "
|
|
"post-freeze retrieval and graph replan are forbidden for this revision."
|
|
)
|
|
pipeline_status = PipelineTerminalStatus(
|
|
"structured_failure",
|
|
primary_failure=e.primary_failure,
|
|
terminal_failure=e.terminal_failure,
|
|
recovery_outcomes=e.recovery_outcomes,
|
|
causal_chain=e.causal_chain,
|
|
)
|
|
break
|
|
except Exception as e:
|
|
logger.error(
|
|
"[Framework] Unhandled pipeline exception: {}",
|
|
type(e).__name__,
|
|
)
|
|
generate_report(
|
|
public_query_reference,
|
|
planner_output,
|
|
decisions,
|
|
output_path=report_path,
|
|
routing_bundles=routing_bundles,
|
|
)
|
|
raise
|
|
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Entry Point
|
|
# ─────────────────────────────────────────────
|
|
return pipeline_status
|
|
|
|
|
|
def _configured_pricing_refs(
|
|
llm_settings: dict,
|
|
*,
|
|
retrieval_policy_path: str | Path | None = None,
|
|
) -> list[str]:
|
|
"""Collect exact configured model IDs without fuzzy or legacy-tag matching."""
|
|
|
|
refs: list[str] = []
|
|
|
|
def add(value) -> None:
|
|
values = value if isinstance(value, (list, tuple)) else [value]
|
|
for item in values:
|
|
text = str(item or "").strip()
|
|
if text and text not in refs:
|
|
refs.append(text)
|
|
|
|
configured_model = llm_settings.get("model", DEFAULT_SYSTEM_MODEL_CHAIN[0])
|
|
add(configured_model)
|
|
add(llm_settings.get("system_model_chain", []))
|
|
add(llm_settings.get("full_generation_baseline_model", configured_model))
|
|
add(llm_settings.get("supplemental_model_ids", []))
|
|
for key in (
|
|
"planner_model",
|
|
"fallback_model",
|
|
"context_compression_model",
|
|
):
|
|
add(llm_settings.get(key))
|
|
add(load_retrieval_policy(retrieval_policy_path).hyde.api_model_id)
|
|
return refs
|
|
|
|
|
|
@bound_pipeline
|
|
async def run_pipeline(
|
|
config: dict,
|
|
user_query: str,
|
|
output_dir: str,
|
|
report_path: str,
|
|
query_diagnostics: dict | None = None,
|
|
task_invocation: PreparedTaskInvocation | None = None,
|
|
network_policy_mode: str = "disabled",
|
|
run_id: str | None = None,
|
|
planner_variant: str = "resource_aware",
|
|
runtime_authority: str = "git",
|
|
*,
|
|
execution_substrate: object | None = None,
|
|
execution_substrate_mode: str = "default",
|
|
max_generation_requests: int | None = None,
|
|
max_embedding_requests: int | None = None,
|
|
require_final_delivery: bool = True,
|
|
method_deadline_monotonic: float | None = None,
|
|
) -> str:
|
|
"""Initialize accounting before paid work and finalize it on every exit."""
|
|
|
|
prepare_retrieval_environment(runtime_authority)
|
|
llm_settings = dict(config.get("llm_settings", {}) or {})
|
|
evaluation_mode = normalize_evaluation_mode(
|
|
llm_settings.get("evaluation_mode")
|
|
)
|
|
retrieval_policy_path = _runtime_policy_path(
|
|
config,
|
|
"retrieval_policy_path",
|
|
"sgar_mvp/config/retrieval_policy.json",
|
|
)
|
|
evaluator_policy_path = _runtime_policy_path(
|
|
config,
|
|
"evaluator_policy_path",
|
|
"sgar_mvp/config/evaluator_policy.json",
|
|
)
|
|
recovery_policy = load_recovery_policy(
|
|
os.path.join(SCRIPT_DIR, "config", "recovery_policy.json")
|
|
)
|
|
evaluator_policy = (
|
|
load_evaluator_policy(evaluator_policy_path)
|
|
if evaluation_mode != "off"
|
|
else None
|
|
)
|
|
required_model_refs = _configured_pricing_refs(
|
|
llm_settings,
|
|
retrieval_policy_path=retrieval_policy_path,
|
|
)
|
|
required_model_refs.append(recovery_policy.full_generation_model_resource_id)
|
|
if evaluator_policy is not None:
|
|
required_model_refs.append(evaluator_policy.model_resource_id)
|
|
catalog = ModelPricingCatalog.from_manifest_file(
|
|
os.path.join(PROJECT_ROOT, "Pool", "resources", "json", "combine.json"),
|
|
required_model_refs=list(dict.fromkeys(required_model_refs)),
|
|
)
|
|
policy = load_model_cost_policy(
|
|
os.path.join(SCRIPT_DIR, "config", "model_cost_policy.json"),
|
|
local_cost_control=llm_settings.get("cost_control"),
|
|
)
|
|
cost_ledger = RunCostLedger(
|
|
catalog=catalog,
|
|
policy=policy,
|
|
output_dir=output_dir,
|
|
run_id=run_id,
|
|
max_generation_requests=max_generation_requests,
|
|
)
|
|
try:
|
|
execution_ledger = RunExecutionLedger(
|
|
output_dir=output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
except Exception:
|
|
cost_ledger.close()
|
|
raise
|
|
try:
|
|
recovery_ledger = RecoveryEventLedger(
|
|
output_dir=output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
except Exception:
|
|
execution_ledger.close()
|
|
cost_ledger.close()
|
|
raise
|
|
try:
|
|
evaluation_ledger = EvaluationEventLedger(
|
|
output_dir=output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
artifact_store = ArtifactLifecycleStore(
|
|
output_dir=output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
context_commit_store = ContextCommitStore(artifact_store=artifact_store)
|
|
system_role_probe_audit = SystemRoleProbeAuditStore(
|
|
output_dir,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
except Exception:
|
|
recovery_ledger.close()
|
|
execution_ledger.close()
|
|
cost_ledger.close()
|
|
raise
|
|
try:
|
|
request_budget = ModelRequestBudget(
|
|
deadline_monotonic=method_deadline_monotonic,
|
|
)
|
|
_atomic_write_json(
|
|
os.path.join(output_dir, "model_request_budget.json"), request_budget.record(),
|
|
)
|
|
with model_request_budget(request_budget), embedding_request_budget(max_embedding_requests) as embedding_budget:
|
|
try:
|
|
result = await _run_pipeline_with_cost_ledger(
|
|
config,
|
|
user_query,
|
|
output_dir,
|
|
report_path,
|
|
cost_ledger,
|
|
execution_ledger,
|
|
recovery_policy,
|
|
recovery_ledger,
|
|
evaluator_policy,
|
|
evaluation_ledger,
|
|
artifact_store,
|
|
context_commit_store,
|
|
system_role_probe_audit=system_role_probe_audit,
|
|
query_diagnostics=query_diagnostics,
|
|
task_invocation=task_invocation,
|
|
network_policy_mode=network_policy_mode,
|
|
planner_variant=planner_variant,
|
|
runtime_authority=runtime_authority,
|
|
execution_substrate=execution_substrate,
|
|
execution_substrate_mode=execution_substrate_mode,
|
|
require_final_delivery=require_final_delivery,
|
|
)
|
|
finally:
|
|
_atomic_write_json(
|
|
os.path.join(output_dir, "embedding_summary.json"),
|
|
embedding_budget.summary(),
|
|
)
|
|
if system_role_probe_audit.status == "in_progress":
|
|
terminal = getattr(result, "failure", None)
|
|
if not isinstance(terminal, TerminalFailureEnvelope):
|
|
terminal = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="system_role_schema_probe",
|
|
failure_code="system_role_schema_probe_audit_incomplete",
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
result = _pipeline_status_from_terminal_failure(terminal)
|
|
system_role_probe_audit.terminate(terminal)
|
|
return result
|
|
except BudgetControlError as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="budget",
|
|
failure_stage="budget_control",
|
|
failure_code=str(getattr(exc, "error_code", "model_cost_limit_reached")),
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
except ModelAccountingError as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="model_accounting",
|
|
failure_code=str(getattr(exc, "error_code", "model_accounting_failure")),
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
except (ModelTransportCapabilityError, ModelResponseContractError) as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="system_role_schema_probe",
|
|
failure_code=str(
|
|
getattr(exc, "error_code", "system_role_schema_probe_contract_invalid")
|
|
),
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
return _pipeline_status_from_terminal_failure(failure)
|
|
except (asyncio.CancelledError, KeyboardInterrupt) as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="interrupted",
|
|
failure_stage="pipeline",
|
|
failure_code="pipeline_interrupted",
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
try:
|
|
system_role_probe_audit.terminate(failure)
|
|
finally:
|
|
raise
|
|
except Exception as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="system_role_schema_probe",
|
|
failure_code="system_role_schema_probe_sequence_aborted",
|
|
exception=exc,
|
|
run_id=cost_ledger.run_id,
|
|
)
|
|
system_role_probe_audit.terminate(failure)
|
|
raise
|
|
finally:
|
|
try:
|
|
execution_summary = execution_ledger.close()
|
|
except ExecutionEventError as exc:
|
|
logger.error(
|
|
"[ResourceRuntime] Failed to persist final summary: {}",
|
|
type(exc).__name__,
|
|
)
|
|
execution_summary = execution_ledger.summary()
|
|
try:
|
|
summary = cost_ledger.close()
|
|
except ModelAccountingError as exc:
|
|
logger.error("[ModelCost] Failed to persist final summary: {}", type(exc).__name__)
|
|
summary = cost_ledger.summary()
|
|
recovery_references = recovery_ledger.summary().get(
|
|
"model_accounting_references", []
|
|
)
|
|
recovery_cost_projection = cost_ledger.operation_cost_summary(
|
|
recovery_references
|
|
)
|
|
try:
|
|
recovery_summary = recovery_ledger.close(
|
|
model_cost_projection=recovery_cost_projection
|
|
)
|
|
except RecoveryPersistenceError as exc:
|
|
logger.error(
|
|
"[Recovery] Failed to persist final summary: {}",
|
|
type(exc).__name__,
|
|
)
|
|
recovery_summary = recovery_ledger.summary()
|
|
try:
|
|
evaluation_summary = evaluation_ledger.close()
|
|
except Exception as exc:
|
|
logger.error(
|
|
"[Evaluation] Failed to persist final summary: {}",
|
|
type(exc).__name__,
|
|
)
|
|
evaluation_summary = evaluation_ledger.summary()
|
|
try:
|
|
artifact_summary, context_summary = artifact_store.ledger.close()
|
|
except Exception as exc:
|
|
logger.error(
|
|
"[Artifacts] Failed to persist final lifecycle summaries: {}",
|
|
type(exc).__name__,
|
|
)
|
|
artifact_summary = artifact_store.ledger.artifact_summary()
|
|
context_summary = artifact_store.ledger.context_summary()
|
|
try:
|
|
upsert_model_cost_section(report_path, summary)
|
|
except Exception as exc:
|
|
logger.error("[ModelCost] Failed to update report section: {}", type(exc).__name__)
|
|
try:
|
|
upsert_execution_section(report_path, execution_summary)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"[ResourceRuntime] Failed to update report section: {}",
|
|
type(exc).__name__,
|
|
)
|
|
try:
|
|
upsert_recovery_section(report_path, recovery_summary)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"[Recovery] Failed to update report section: {}",
|
|
type(exc).__name__,
|
|
)
|
|
try:
|
|
from sgar_mvp.src.reporter import upsert_evaluation_artifact_section
|
|
|
|
upsert_evaluation_artifact_section(
|
|
report_path,
|
|
evaluation_summary=evaluation_summary,
|
|
artifact_summary=artifact_summary,
|
|
context_summary=context_summary,
|
|
)
|
|
except Exception as exc:
|
|
logger.error(
|
|
"[Evaluation] Failed to update report section: {}",
|
|
type(exc).__name__,
|
|
)
|
|
|
|
|
|
def _terminal_exit_code(terminal_status: str) -> int:
|
|
status = str(terminal_status or "").strip().lower()
|
|
if status == "succeeded":
|
|
return 0
|
|
if status == "interrupted":
|
|
return 130
|
|
return 1
|
|
|
|
|
|
def _finalized_terminal_state(
|
|
pipeline_status: str, provisional_status: str, manifest: Any,
|
|
) -> tuple[str, str]:
|
|
"""Use final audit results for both the CLI summary and process exit."""
|
|
final_status = str(manifest.status)
|
|
if final_status != provisional_status:
|
|
pipeline_status = "structured_failure"
|
|
return pipeline_status, final_status
|
|
|
|
|
|
def _git_checkout_runtime_identity(project_root: str | Path = PROJECT_ROOT) -> dict[str, Any]:
|
|
"""Describe the imported checkout without manufacturing a release attestation."""
|
|
|
|
root = Path(project_root).resolve()
|
|
|
|
def git(*arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["git", *arguments],
|
|
cwd=root,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="strict",
|
|
capture_output=True,
|
|
check=check,
|
|
)
|
|
|
|
try:
|
|
branch = git("branch", "--show-current").stdout.strip()
|
|
head = git("rev-parse", "HEAD").stdout.strip()
|
|
tree = git("rev-parse", "HEAD^{tree}").stdout.strip()
|
|
unstaged = git("diff", "--quiet", "--no-ext-diff", check=False).returncode
|
|
staged = git("diff", "--cached", "--quiet", "--no-ext-diff", check=False).returncode
|
|
except (OSError, subprocess.SubprocessError) as exc:
|
|
raise RuntimeError("git_runtime_identity_unavailable") from exc
|
|
if unstaged not in {0, 1} or staged not in {0, 1}:
|
|
raise RuntimeError("git_runtime_dirty_state_unavailable")
|
|
dirty = bool(unstaged or staged)
|
|
return {
|
|
"source_authority": "git_worktree",
|
|
"branch": branch,
|
|
"git_head": head,
|
|
"git_tree": tree,
|
|
"tracked_dirty": dirty,
|
|
"runtime_classification": (
|
|
"experimental_uncommitted" if dirty else "git_commit"
|
|
),
|
|
"release_attestation": "not_requested",
|
|
}
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
bootstrap_config = load_config(args.config, resolve_secrets=False) or {}
|
|
runtime_settings = bootstrap_config.get("runtime_settings") or {}
|
|
configure_runtime(args, config=bootstrap_config, project_root=Path(PROJECT_ROOT))
|
|
auto_refresh = bool(runtime_settings.get("auto_refresh_control_receipt", False))
|
|
default_profile = str(runtime_settings.get("model_profile_path") or "").replace("\\", "/")
|
|
is_default_profile = default_profile == "sgar_mvp/config/model_profile.json"
|
|
if (
|
|
args.runtime_authority == "git"
|
|
and auto_refresh
|
|
and is_default_profile
|
|
and os.environ.get("SGAR_SKIP_AUTO_CONTROL_PROBE") != "1"
|
|
):
|
|
refresh_root = Path(tempfile.mkdtemp(prefix="sgar-control-probe-refresh-"))
|
|
commands = (
|
|
[sys.executable, "-m", "sgar_mvp.scripts.sync_default_model_profile"],
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"sgar_mvp.scripts.refresh_local_control_probe_receipt",
|
|
"--output-dir",
|
|
str(refresh_root),
|
|
"--config",
|
|
str(args.config),
|
|
],
|
|
)
|
|
for command in commands:
|
|
try:
|
|
subprocess.run(command, cwd=PROJECT_ROOT, check=True)
|
|
except (OSError, subprocess.CalledProcessError) as exc:
|
|
raise SystemExit("SGAR control model profile refresh failed") from exc
|
|
if args.runtime_authority == "git" and (
|
|
args.source_seal or args.sealed_local_validation
|
|
):
|
|
raise SystemExit(
|
|
"--source-seal/--sealed-local-validation require --runtime-authority release"
|
|
)
|
|
if args.runtime_authority == "release" and bool(args.source_seal) != bool(
|
|
args.sealed_local_validation
|
|
):
|
|
raise SystemExit(
|
|
"--source-seal and --sealed-local-validation must be supplied together"
|
|
)
|
|
if args.sealed_local_validation:
|
|
if os.environ.get("SGAR_SEALED_LOCAL_VALIDATION") != "1":
|
|
raise SystemExit("sealed local validation environment is not active")
|
|
configured_seal = os.environ.get("SGAR_SOURCE_SEAL_PATH")
|
|
if not configured_seal or Path(configured_seal).resolve() != Path(
|
|
args.source_seal
|
|
).resolve():
|
|
raise SystemExit("sealed local validation source identity mismatch")
|
|
if args.run_dir and args.output_dir:
|
|
raise SystemExit("--run-dir and deprecated --output-dir cannot be combined")
|
|
explicit_run_dir = args.run_dir or args.output_dir
|
|
try:
|
|
workspace = create_run_workspace(
|
|
output_root=Path(args.output_root),
|
|
explicit_run_dir=(Path(explicit_run_dir) if explicit_run_dir else None),
|
|
)
|
|
except RunWorkspaceError as exc:
|
|
raise SystemExit(f"SGAR run workspace rejected: {type(exc).__name__}") from exc
|
|
|
|
output_dir = str(workspace.path())
|
|
bootstrap_request_id = uuid.uuid4().hex
|
|
bootstrap_identity = canonical_sha256(
|
|
{
|
|
"protocol": "sgar-task-invocation-v1",
|
|
"request_id": bootstrap_request_id,
|
|
"status": "input_snapshot_pending",
|
|
}
|
|
)
|
|
manifest_store = RunManifestStore(
|
|
run_dir=workspace.path(),
|
|
run_id=workspace.run_id,
|
|
request_id=bootstrap_request_id,
|
|
invocation_sha256=bootstrap_identity,
|
|
)
|
|
|
|
try:
|
|
resolved_request = _resolve_task_request_arguments(
|
|
args,
|
|
run_dir=workspace.path(),
|
|
)
|
|
prepared_invocation = prepare_task_invocation(
|
|
query=resolved_request.exact_query,
|
|
input_specs=resolved_request.input_specs,
|
|
run_dir=workspace.path(),
|
|
request_id=resolved_request.request_id,
|
|
final_deliverable_contract=resolved_request.final_deliverable_contract,
|
|
public_context_descriptors=resolved_request.public_context_descriptors,
|
|
allowed_public_input_roots=(
|
|
resolved_request.allowed_public_input_roots
|
|
),
|
|
project_root=Path(PROJECT_ROOT),
|
|
)
|
|
manifest_store.bind_invocation(
|
|
request_id=prepared_invocation.invocation.request_id,
|
|
invocation_sha256=prepared_invocation.invocation.invocation_sha256,
|
|
identities={
|
|
"public_input_snapshot_sha256": (
|
|
prepared_invocation.invocation.input_snapshot_sha256
|
|
),
|
|
"task_invocation_protocol": prepared_invocation.invocation.protocol,
|
|
"request_source_sha256": resolved_request.request_source_sha256,
|
|
},
|
|
)
|
|
except (TaskInvocationError, OSError, ValueError) as exc:
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="task_invocation",
|
|
failure_code="task_invocation_invalid",
|
|
exception=exc,
|
|
),
|
|
)
|
|
raise SystemExit("SGAR task invocation rejected before paid work") from exc
|
|
|
|
run_root = workspace.path().resolve()
|
|
|
|
def isolated_output_path(value: str | None, default_name: str) -> Path:
|
|
candidate = Path(value).absolute() if value else run_root / default_name
|
|
try:
|
|
candidate.resolve(strict=False).relative_to(run_root)
|
|
except ValueError as exc:
|
|
raise RunWorkspaceError("run_output_path_outside_workspace") from exc
|
|
if candidate.exists():
|
|
raise RunWorkspaceError("run_output_path_exists")
|
|
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
return candidate
|
|
|
|
try:
|
|
report_path_obj = isolated_output_path(args.report_path, "experiment_report.md")
|
|
log_path_obj = isolated_output_path(args.log_path, "pipeline.log")
|
|
except RunWorkspaceError as exc:
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="run_workspace",
|
|
failure_code="run_output_path_invalid",
|
|
exception=exc,
|
|
),
|
|
)
|
|
raise SystemExit("SGAR output path rejected before logging") from exc
|
|
|
|
report_path = str(report_path_obj)
|
|
log_path = str(log_path_obj)
|
|
logger.add(log_path, mode="x", encoding="utf-8", level="INFO")
|
|
progress_path = run_root / "progress.log"
|
|
terminal_progress.configure(progress_path=(
|
|
progress_path if progress_path not in {log_path_obj.resolve(), report_path_obj.resolve()} else None
|
|
))
|
|
terminal_progress.location("Run directory", run_root, run_root)
|
|
terminal_progress.location("Full log", log_path, run_root)
|
|
manifest_store.mark_running()
|
|
logger.info("=" * 60)
|
|
|
|
framework_source_clean = False
|
|
if args.runtime_authority == "git":
|
|
try:
|
|
git_identity = _git_checkout_runtime_identity(PROJECT_ROOT)
|
|
except Exception as exc:
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="runtime_authority",
|
|
failure_code="git_runtime_identity_invalid",
|
|
exception=exc,
|
|
),
|
|
)
|
|
return 1
|
|
framework_source_clean = not bool(git_identity["tracked_dirty"])
|
|
manifest_store.update_identities(
|
|
{"runtime_authority": "git", **git_identity}
|
|
)
|
|
else:
|
|
manifest_store.update_identities(
|
|
{
|
|
"runtime_authority": "release",
|
|
"source_authority": "release_attestation",
|
|
}
|
|
)
|
|
logger.info("S-GAR MVP — End-to-End Pipeline")
|
|
logger.info("=" * 60)
|
|
|
|
raw_config = load_config(args.config, resolve_secrets=False)
|
|
try:
|
|
if raw_config is None:
|
|
raise FormalSecretPolicyError("formal_configuration_missing")
|
|
secret_policy = validate_formal_secret_config(raw_config)
|
|
except FormalSecretPolicyError as exc:
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="secret_policy",
|
|
failure_code=exc.failure_code,
|
|
exception=exc,
|
|
),
|
|
)
|
|
logger.error("Formal secret policy rejected the runtime configuration.")
|
|
return 1
|
|
|
|
config = load_config(args.config)
|
|
if not config or config.get("llm_key") == "your_api_key_here":
|
|
logger.error("Invalid config.json — please set a valid llm_key.")
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="configuration",
|
|
failure_code="llm_configuration_invalid",
|
|
),
|
|
)
|
|
return 1
|
|
manifest_store.set_private_portability_needles(
|
|
secret_values=(str(config.get("llm_key") or ""),)
|
|
)
|
|
manifest_store.update_identities(
|
|
{
|
|
"formal_secret_policy_protocol": secret_policy.protocol,
|
|
"formal_secret_policy_sha256": secret_policy.policy_sha256,
|
|
}
|
|
)
|
|
|
|
if args.runtime_authority == "git":
|
|
try:
|
|
base_url = str(
|
|
(config.get("llm_settings") or {}).get(
|
|
"base_url", "https://api.openai.com/v1"
|
|
)
|
|
)
|
|
endpoint_identity = production_model_endpoint_identity(base_url=base_url)
|
|
runtime_identity = build_retrieval_runtime_identity(
|
|
project_root=PROJECT_ROOT,
|
|
provider_compatibility=_provider_compatibility_projection(base_url),
|
|
provider_endpoint_identity_sha256=endpoint_identity.identity_sha256,
|
|
require_release_sealed=False,
|
|
honor_release_environment=False,
|
|
)
|
|
ready_state = load_applied_model_ready_state(
|
|
PROJECT_ROOT,
|
|
expected_endpoint_identity_sha256=endpoint_identity.identity_sha256,
|
|
)
|
|
active_manifest = json.loads(
|
|
(
|
|
Path(PROJECT_ROOT)
|
|
/ "Pool"
|
|
/ "index_meta"
|
|
/ "index_build_manifest.json"
|
|
).read_text(encoding="utf-8-sig")
|
|
)
|
|
manifest_store.update_identities(
|
|
{
|
|
"model_ready_state_sha256": ready_state.health_sha256,
|
|
"model_ready_count": len(ready_state.models),
|
|
"provider_endpoint_identity_sha256": endpoint_identity.identity_sha256,
|
|
"retrieval_runtime_identity_sha256": runtime_identity.identity_sha256,
|
|
"active_retrieval_generation_id": active_manifest.get(
|
|
"generation_id"
|
|
),
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="runtime_authority",
|
|
failure_code="git_runtime_authority_invalid",
|
|
exception=exc,
|
|
),
|
|
)
|
|
return 1
|
|
|
|
if args.runtime_authority == "git":
|
|
manifest_store.update_phase("framework_conformance", "not_requested")
|
|
manifest_store.update_identities(
|
|
{"release_attestation": "not_requested"}
|
|
)
|
|
else:
|
|
manifest_store.update_phase("framework_conformance", "running")
|
|
try:
|
|
base_url = str(
|
|
(config.get("llm_settings") or {}).get(
|
|
"base_url", "https://api.openai.com/v1"
|
|
)
|
|
)
|
|
conformance = run_production_conformance(
|
|
prepared_invocation=prepared_invocation,
|
|
request_source_sha256=resolved_request.request_source_sha256,
|
|
run_dir=workspace.path(),
|
|
config=raw_config,
|
|
network_policy_mode=args.network_policy,
|
|
project_root=PROJECT_ROOT,
|
|
provider_compatibility=_provider_compatibility_projection(base_url),
|
|
require_docker=True,
|
|
require_source_clean=True,
|
|
source_seal_path=args.source_seal,
|
|
)
|
|
manifest_store.update_identities(
|
|
{
|
|
"framework_conformance_protocol": conformance.get("protocol"),
|
|
"framework_conformance_sha256": conformance.get("report_sha256"),
|
|
**dict(conformance.get("identities") or {}),
|
|
}
|
|
)
|
|
framework_source_clean = bool(
|
|
(conformance.get("checks") or {})
|
|
.get("framework_source", {})
|
|
.get("framework_source_clean")
|
|
)
|
|
if conformance.get("valid") is not True:
|
|
conformance_errors = {
|
|
str(item) for item in (conformance.get("errors") or ())
|
|
}
|
|
failure_code = (
|
|
"internal_metadata_layout_invalid"
|
|
if "internal_metadata_layout_invalid" in conformance_errors
|
|
else "production_conformance_invalid"
|
|
)
|
|
manifest_store.update_phase("framework_conformance", "failed")
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="framework_conformance",
|
|
failure_code=failure_code,
|
|
),
|
|
framework_source_clean=bool(
|
|
(conformance.get("checks") or {})
|
|
.get("framework_source", {})
|
|
.get("framework_source_clean")
|
|
),
|
|
)
|
|
logger.error("Production conformance failed before paid work.")
|
|
return 1
|
|
manifest_store.update_phase("framework_conformance", "passed")
|
|
except KeyboardInterrupt as exc:
|
|
manifest_store.update_phase("framework_conformance", "interrupted")
|
|
manifest_store.terminal(
|
|
"interrupted",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="interrupted",
|
|
failure_stage="framework_conformance",
|
|
failure_code="user_interrupted",
|
|
exception=exc,
|
|
),
|
|
framework_source_clean=False,
|
|
)
|
|
return 130
|
|
except Exception as exc:
|
|
manifest_store.update_phase("framework_conformance", "failed")
|
|
manifest_store.terminal(
|
|
"framework_failure",
|
|
primary_failure=sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="framework_conformance",
|
|
failure_code="production_conformance_exception",
|
|
exception=exc,
|
|
),
|
|
framework_source_clean=False,
|
|
)
|
|
logger.error(
|
|
"Production conformance raised before paid work: {}",
|
|
type(exc).__name__,
|
|
)
|
|
return 1
|
|
|
|
user_query, query_diagnostics = diagnose_and_repair_query_text(
|
|
prepared_invocation.invocation.query
|
|
)
|
|
if query_diagnostics.get("query_encoding_warning"):
|
|
logger.warning(
|
|
"[Input] Query encoding warning: {} | repair_applied={}",
|
|
query_diagnostics.get("reason"),
|
|
query_diagnostics.get("repair_applied"),
|
|
)
|
|
|
|
logger.info("Run ID: {}", workspace.run_id)
|
|
logger.info("Explicit public inputs: {}", len(prepared_invocation.invocation.public_inputs))
|
|
|
|
terminal_status = "framework_failure"
|
|
primary_failure = None
|
|
terminal_manifest_failure = None
|
|
recovery_outcome = None
|
|
causal_chain: tuple[Mapping[str, object], ...] = ()
|
|
try:
|
|
manifest_store.update_phase("pipeline", "running")
|
|
pipeline_status = asyncio.run(
|
|
run_pipeline(
|
|
config,
|
|
user_query,
|
|
output_dir,
|
|
report_path,
|
|
query_diagnostics=query_diagnostics,
|
|
task_invocation=prepared_invocation,
|
|
network_policy_mode=args.network_policy,
|
|
run_id=workspace.run_id,
|
|
planner_variant=args.planner_variant,
|
|
runtime_authority=args.runtime_authority,
|
|
max_generation_requests=args.max_generation_requests,
|
|
method_deadline_monotonic=(
|
|
time.monotonic() + args.deadline_sec
|
|
if args.deadline_sec is not None else None
|
|
),
|
|
)
|
|
)
|
|
if pipeline_status in {"complete_success", "success_with_warnings"}:
|
|
terminal_status = "succeeded"
|
|
recovery_outcome = _pipeline_recovery_outcome_projection(
|
|
getattr(pipeline_status, "recovery_outcomes", ())
|
|
)
|
|
causal_chain = tuple(
|
|
dict(item)
|
|
for item in (getattr(pipeline_status, "causal_chain", ()) or ())
|
|
if isinstance(item, Mapping)
|
|
)
|
|
manifest_store.update_phase("pipeline", "succeeded")
|
|
else:
|
|
projection = _pipeline_terminal_projection(pipeline_status)
|
|
terminal_status = str(projection["status"])
|
|
manifest_store.update_phase("pipeline", terminal_status)
|
|
primary_failure = projection["primary_failure"]
|
|
terminal_manifest_failure = projection["terminal_failure"]
|
|
recovery_outcome = projection["recovery_outcome"]
|
|
causal_chain = projection["causal_chain"]
|
|
except BudgetControlError as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="budget",
|
|
failure_stage="budget_control",
|
|
failure_code=str(getattr(exc, "error_code", "model_cost_limit_reached")),
|
|
exception=exc,
|
|
run_id=workspace.run_id,
|
|
)
|
|
pipeline_status = _pipeline_status_from_terminal_failure(failure)
|
|
projection = _pipeline_terminal_projection(pipeline_status)
|
|
terminal_status = str(projection["status"])
|
|
primary_failure = projection["primary_failure"]
|
|
terminal_manifest_failure = projection["terminal_failure"]
|
|
causal_chain = projection["causal_chain"]
|
|
except ModelAccountingError as exc:
|
|
logger.error(
|
|
"[ModelCost] Pricing/accounting configuration failed before a paid call: {}",
|
|
type(exc).__name__,
|
|
)
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="framework",
|
|
failure_stage="model_accounting",
|
|
failure_code=str(
|
|
getattr(exc, "error_code", "model_accounting_configuration_failed")
|
|
),
|
|
exception=exc,
|
|
run_id=workspace.run_id,
|
|
)
|
|
pipeline_status = _pipeline_status_from_terminal_failure(failure)
|
|
projection = _pipeline_terminal_projection(pipeline_status)
|
|
terminal_status = str(projection["status"])
|
|
primary_failure = projection["primary_failure"]
|
|
terminal_manifest_failure = projection["terminal_failure"]
|
|
causal_chain = projection["causal_chain"]
|
|
except (asyncio.CancelledError, KeyboardInterrupt) as exc:
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility="interrupted",
|
|
failure_stage="pipeline",
|
|
failure_code=(
|
|
"pipeline_cancelled"
|
|
if isinstance(exc, asyncio.CancelledError)
|
|
else "user_interrupted"
|
|
),
|
|
exception=exc,
|
|
run_id=workspace.run_id,
|
|
)
|
|
pipeline_status = _pipeline_status_from_terminal_failure(failure)
|
|
projection = _pipeline_terminal_projection(pipeline_status)
|
|
terminal_status = str(projection["status"])
|
|
primary_failure = projection["primary_failure"]
|
|
terminal_manifest_failure = projection["terminal_failure"]
|
|
causal_chain = projection["causal_chain"]
|
|
except Exception as exc:
|
|
retryable, transport_code = classify_transport_exception(exc)
|
|
if transport_code != "provider_non_transport_error":
|
|
responsibility = "infrastructure"
|
|
failure_stage = "provider_transport"
|
|
failure_code = transport_code
|
|
elif isinstance(exc, InternalMetadataLayoutError):
|
|
responsibility = "framework"
|
|
failure_stage = "internal_metadata_layout"
|
|
failure_code = exc.failure_code
|
|
elif isinstance(exc, RecoveryControlError):
|
|
responsibility = "framework"
|
|
failure_stage = "recovery_configuration"
|
|
failure_code = "recovery_configuration_failed"
|
|
elif isinstance(exc, (EvaluationRuntimeError, ArtifactLifecycleError)):
|
|
responsibility = "framework"
|
|
failure_stage = "evaluation_artifact_runtime"
|
|
failure_code = "evaluation_artifact_runtime_failed"
|
|
elif isinstance(exc, RetrievalRuntimeError):
|
|
projected_responsibility = str(
|
|
getattr(exc, "failure_responsibility", "framework")
|
|
)
|
|
responsibility = (
|
|
projected_responsibility
|
|
if projected_responsibility
|
|
in {"framework", "infrastructure", "research", "budget"}
|
|
else "framework"
|
|
)
|
|
failure_stage = "candidate_model_liveness"
|
|
failure_code = str(
|
|
getattr(exc, "error_code", "pipeline_subsystem_contract_failed")
|
|
)
|
|
elif isinstance(exc, ExecutionEventError):
|
|
responsibility = "framework"
|
|
failure_stage = "pipeline_subsystem"
|
|
failure_code = "pipeline_subsystem_contract_failed"
|
|
else:
|
|
responsibility = "framework"
|
|
failure_stage = "pipeline_unclassified_exception"
|
|
exception_token = re.sub(
|
|
r"[^a-z0-9]+",
|
|
"_",
|
|
type(exc).__name__.strip().lower(),
|
|
).strip("_") or "unknown"
|
|
failure_code = f"pipeline_unclassified_{exception_token}"
|
|
failure = TerminalFailureEnvelope.create(
|
|
responsibility=responsibility,
|
|
failure_stage=failure_stage,
|
|
failure_code=failure_code,
|
|
exception=exc,
|
|
retryable=retryable if responsibility == "infrastructure" else False,
|
|
run_id=workspace.run_id,
|
|
)
|
|
pipeline_status = _pipeline_status_from_terminal_failure(failure)
|
|
projection = _pipeline_terminal_projection(pipeline_status)
|
|
terminal_status = str(projection["status"])
|
|
primary_failure = projection["primary_failure"]
|
|
terminal_manifest_failure = projection["terminal_failure"]
|
|
causal_chain = projection["causal_chain"]
|
|
finally:
|
|
try:
|
|
manifest_store.update_identities(
|
|
{
|
|
**_model_liveness_manifest_identity(workspace.path()),
|
|
**_dag_edge_manifest_identity(workspace.path()),
|
|
**_planner_attempt_manifest_identity(workspace.path()),
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
manifest_store.add_secondary_audit_failure(
|
|
sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="run_manifest",
|
|
failure_code="terminal_evidence_manifest_projection_failed",
|
|
exception=exc,
|
|
).model_dump(mode="json")
|
|
)
|
|
if terminal_status == "succeeded":
|
|
primary_failure = sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="run_manifest",
|
|
failure_code="terminal_evidence_manifest_projection_failed",
|
|
exception=exc,
|
|
)
|
|
terminal_manifest_failure = primary_failure
|
|
terminal_status = "framework_failure"
|
|
ledger_evidence = collect_run_ledger_evidence(
|
|
workspace.path(),
|
|
expected_run_id=workspace.run_id,
|
|
)
|
|
if terminal_status != "interrupted" and any(
|
|
ledger_evidence["unmatched_calls"].values()
|
|
):
|
|
audit_failure = sanitized_run_failure(
|
|
responsibility="framework",
|
|
failure_stage="run_manifest",
|
|
failure_code=(
|
|
"terminal_run_ledger_audit_invalid"
|
|
if ledger_evidence["unmatched_calls"].get(
|
|
"ledger_audit_errors", 0
|
|
)
|
|
else "terminal_run_has_unmatched_calls"
|
|
),
|
|
)
|
|
if primary_failure is None:
|
|
primary_failure = audit_failure
|
|
terminal_manifest_failure = audit_failure
|
|
terminal_status = "framework_failure"
|
|
else:
|
|
manifest_store.add_secondary_audit_failure(
|
|
audit_failure.model_dump(mode="json")
|
|
)
|
|
causal_chain = (
|
|
*causal_chain,
|
|
{
|
|
"sequence": len(causal_chain),
|
|
"kind": "terminal_audit_failure",
|
|
"subtask_id": "",
|
|
"responsibility": audit_failure.responsibility,
|
|
"failure_stage": audit_failure.failure_stage,
|
|
"failure_code": audit_failure.failure_code,
|
|
"message_sha256": audit_failure.message_sha256,
|
|
},
|
|
)
|
|
manifest_store.update_phase("pipeline", terminal_status)
|
|
final_manifest = manifest_store.terminal(
|
|
terminal_status,
|
|
primary_failure=primary_failure,
|
|
terminal_failure=terminal_manifest_failure,
|
|
recovery_outcome=recovery_outcome,
|
|
causal_chain=causal_chain,
|
|
ledger_hashes=ledger_evidence["ledger_hashes"],
|
|
unmatched_calls=ledger_evidence["unmatched_calls"],
|
|
framework_source_clean=framework_source_clean,
|
|
)
|
|
pipeline_status, terminal_status = _finalized_terminal_state(
|
|
pipeline_status, terminal_status, final_manifest,
|
|
)
|
|
final_message = _terminal_pipeline_log_lines(pipeline_status)[0]
|
|
if pipeline_status == "structured_failure":
|
|
logger.error(final_message)
|
|
else:
|
|
logger.info(final_message)
|
|
terminal_progress.final_summary(pipeline_status, run_root, report_path, log_path, manifest=final_manifest)
|
|
return _terminal_exit_code(terminal_status)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|