ChatDev/execorchestra/retrieve.py
2026-11-05 16:22:12 +08:00

1787 lines
68 KiB
Python

"""HyDE-enhanced, typed dual-path retrieval for S-GAR."""
from __future__ import annotations
import json
import hashlib
import logging
import os
import pickle
import re
import threading
import time
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Mapping, Sequence, Tuple
import certifi
import faiss
import numpy as np
from retrieval_profiles import PROFILE_VERSION
from sgar_mvp.src.embedding_runtime import (
EmbeddingRuntimeConfigV1,
IndexMetaV2,
LocalEmbeddingEncoder,
embedding_config_from_metadata,
prepare_query_texts,
)
from sgar_mvp.src.internal_language import prompt_file_text
from sgar_mvp.src.retrieval_policy import (
RETRIEVAL_STRATEGIES,
load_retrieval_policy,
)
from sgar_mvp.src.model_accounting import (
BudgetControlError,
ModelAccountingError,
RunCostLedger,
)
from sgar_mvp.src.proxy_retry import wait_for_proxy_retry
from sgar_mvp.src.model_transport import (
ModelTransportError,
ProviderEndpointIdentity,
SyncModelTransportPort,
classify_transport_exception,
legacy_sync_transport_from_client,
model_request_sha256,
require_sync_model_transport,
)
from sgar_mvp.src.model_response_contracts import (
StructuredResponseModeInput,
normalize_portable_wire_instance,
normalize_structured_response_mode,
strict_json_loads,
structured_response_format,
system_role_requirement,
validate_json_schema_instance,
)
from sgar_mvp.src.retrieval_lifecycle import (
LocalEmbeddingIdentity,
RetrievalLifecycleError,
)
from sgar_mvp.src.profiler_protocol import (
PROFILER_NORMAL_OUTPUT_CAP,
PROFILER_TRUNCATION_RETRY_OUTPUT_CAP,
ProfilerInputEnvelopeV1,
ProfilerOutputV2,
)
from sgar_mvp.src.runtime_policy_context import active_retrieval_policy_path
from sgar_mvp.src.provider_reasoning import observe_provider_reasoning
from sgar_mvp.src.release_source_seal import (
load_and_verify_source_seal,
source_seal_reference,
)
PROJECT_ROOT = Path(__file__).resolve().parent
INDEX_DIR = Path(
os.environ.get("SGAR_INDEX_DIR", str(PROJECT_ROOT / "Pool" / "index_meta"))
).resolve()
CAP_INDEX_FILE = INDEX_DIR / "faiss_cap.index"
CON_INDEX_FILE = INDEX_DIR / "faiss_con.index"
METADATA_FILE = INDEX_DIR / "resource_meta.pkl"
MODEL_HEALTH_FILE = PROJECT_ROOT / "sgar_mvp" / "config" / "model_health.json"
_BOUND_POLICY_PATH = active_retrieval_policy_path()
_POLICY_OVERRIDE = str(_BOUND_POLICY_PATH) if _BOUND_POLICY_PATH else os.environ.get("SGAR_RETRIEVAL_POLICY_PATH")
if (
_POLICY_OVERRIDE
and _BOUND_POLICY_PATH is None
and os.environ.get("SGAR_SEALED_LOCAL_VALIDATION") != "1"
):
raise RuntimeError("retrieval_policy_environment_override_requires_sealed_validation")
RETRIEVAL_POLICY = (
load_retrieval_policy(Path(_POLICY_OVERRIDE).resolve())
if _POLICY_OVERRIDE
else load_retrieval_policy()
)
EMBED_MODEL = RETRIEVAL_POLICY.embedding_model
BGE_PREFIX = RETRIEVAL_POLICY.bge_prefix
HYDE_MODEL = os.environ.get("SGAR_HYDE_MODEL", RETRIEVAL_POLICY.hyde.api_model_id)
HYDE_BASE_URL = "https://api.example.invalid/v1"
HYDE_PROFILE_VERSION = RETRIEVAL_POLICY.hyde.prompt_version
TOP_N = 10
TOP_K = 3
TYPE_WEIGHTS: Dict[str, Tuple[float, float]] = RETRIEVAL_POLICY.weights()
# Explicit mode for retrieval quality audits. Production remains capability-only
# until a local policy is deliberately promoted after the quality gate.
SHADOW_RETRIEVAL_STRATEGY = "capability_constraint_weighted"
os.environ["SSL_CERT_FILE"] = certifi.where()
os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
PROFILER_SYSTEM_PROMPT = prompt_file_text("profiler_system.txt")
ENGLISH_PROFILER_PROMPT_VERSION = "ideal-resource-profiler-en-v3"
def _profiler_prompt_for_version(
prompt_version: str,
) -> str:
if prompt_version == ENGLISH_PROFILER_PROMPT_VERSION:
return PROFILER_SYSTEM_PROMPT
raise HyDEGenerationError(
"profiler_prompt_version_unknown",
responsibility="framework",
response_received=False,
)
@dataclass(frozen=True)
class QueryProfile:
capability_text: str
constraint_text: str
capability_vector: List[float]
constraint_vector: List[float]
raw_query_text: str = ""
raw_query_vector: List[float] | None = None
hard_requirements: Dict[str, Any] = field(default_factory=dict)
concatenated_vector: List[float] | None = None
profile_version: str = "typed-query-v1"
generation_metadata: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
return {
"capability_text": self.capability_text,
"constraint_text": self.constraint_text,
"capability_vector": self.capability_vector,
"constraint_vector": self.constraint_vector,
"raw_query_text": self.raw_query_text,
"raw_query_vector": self.raw_query_vector,
"hard_requirements": self.hard_requirements,
"concatenated_vector": self.concatenated_vector,
"profile_version": self.profile_version,
"generation_metadata": self.generation_metadata,
}
class HyDEGenerationError(RuntimeError):
"""Structured terminal failure from one semantic HyDE generation."""
def __init__(
self,
failure_code: str,
*,
responsibility: str,
response_received: bool,
transport_attempts: int = 0,
request_sha256: str | None = None,
) -> None:
super().__init__(failure_code)
self.failure_code = str(failure_code)
self.failure_responsibility = str(responsibility)
self.response_received = bool(response_received)
self.transport_attempts = int(transport_attempts)
self.request_sha256 = request_sha256
self.retryable = False
HYDE_PROMPT = _profiler_prompt_for_version(HYDE_PROFILE_VERSION)
def _profiler_output_is_english(
output: ProfilerOutputV2,
source_text: str,
) -> bool:
"""Require English prose while permitting literal CJK identifiers from input."""
combined = f"{output.capability_text}\n{output.constraint_text}"
combined += f"\n{output.think}"
cjk_sequences = re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+", combined)
if any(sequence not in source_text for sequence in cjk_sequences):
return False
latin_count = sum(character.isascii() and character.isalpha() for character in combined)
cjk_count = sum(len(sequence) for sequence in cjk_sequences)
return latin_count >= 20 and latin_count >= cjk_count * 4
def _profiler_think_is_valid(output: ProfilerOutputV2) -> bool:
"""Require a useful rationale; leakage is checked across every field below."""
return bool(output.think.strip())
def _profiler_output_is_safe(
output: ProfilerOutputV2, source_text: str = ""
) -> bool:
"""Apply the no-runtime/no-pool leakage rule to every generated field."""
resource_id_pattern = re.compile(
r"(?i)\b(?:model|tool|skill|agent|resource|device)\.[a-z0-9][a-z0-9_.:-]*"
)
fields = (output.capability_text, output.constraint_text, output.think)
forbidden = (
"resource_id",
"retrieval score",
)
source = source_text.casefold()
# Task-declared file names can share a resource-type prefix (model.weights,
# tool.py). They describe authoritative deliverables, not pool selections.
declared_file_names = {
token.replace("\\", "/").rsplit("/", 1)[-1].rstrip(".,;:!?)]}{").casefold()
for token in re.findall(r"[^\s<>\"'`]*[\\/][^\s<>\"'`]+", source_text)
}
for value in fields:
text = str(value or "").strip()
lowered = text.casefold()
if (
not text
or any(term in lowered for term in forbidden)
or any(
match.group(0).rstrip(".,;:!?)").casefold() not in declared_file_names
for match in resource_id_pattern.finditer(text)
)
):
return False
# Source URLs and paths can be part of the task's authoritative input.
# Reject invented locations without suppressing declared constraints.
locations = re.findall(r"(?i)https?://[^\s<>\"']+|(?:^|\s)[a-z]:[\\/][^\s<>\"']+", text)
if any(location.strip().rstrip(".,;:!?)").casefold() not in source for location in locations):
return False
return True
def get_profiler_transport() -> SyncModelTransportPort:
"""Return the shared exact Chat Completions transport for Profiler calls."""
_engine.ensure_transport_loaded()
legacy_client = _engine.llm_client
if legacy_client is None:
raise HyDEGenerationError(
"hyde_client_unavailable",
responsibility="framework",
response_received=False,
)
base_url = (
_load_env_value("LLM_BASE_URL", "OPENAI_BASE_URL") or HYDE_BASE_URL
).rstrip("/")
endpoint = ProviderEndpointIdentity.create(
provider="openai_compatible",
base_url=base_url,
credential_environment_variable="LLM_API_KEY",
timeout_seconds=60.0,
)
return legacy_sync_transport_from_client(
client=legacy_client,
endpoint_identity=endpoint,
)
def _load_env_value(*names: str) -> str:
dotenv_values: Dict[str, str] = {}
env_path = PROJECT_ROOT / ".env"
if env_path.exists():
for line in env_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
name, value = line.split("=", 1)
dotenv_values[name.strip()] = value.strip()
for name in names:
value = os.environ.get(name, "").strip() or dotenv_values.get(name, "")
if value:
return value
return ""
def _resource_type(raw: Mapping[str, Any]) -> str:
legacy = raw.get("type")
nested = legacy.get("resource_type") if isinstance(legacy, Mapping) else None
return str(raw.get("resource_type") or nested or "Unknown")
def _load_unavailable_model_ids() -> set[str]:
if not MODEL_HEALTH_FILE.exists():
return set()
try:
payload = json.loads(MODEL_HEALTH_FILE.read_text(encoding="utf-8-sig"))
except Exception:
return set()
unavailable = {str(item) for item in payload.get("unavailable_model_ids", [])}
for item in payload.get("models", []):
if isinstance(item, Mapping) and item.get("status") == "unavailable":
unavailable.add(str(item.get("model_id") or ""))
return {item for item in unavailable if item}
class IndexMetadataRuntime:
"""Thread-safe frozen index lifecycle with no encoder or network client."""
def __init__(self) -> None:
self._lock = threading.RLock()
self._loaded = False
self.cap_index: faiss.Index | None = None
self.con_index: faiss.Index | None = None
self.metadata: Dict[str, Any] | None = None
self.cap_matrix: np.ndarray | None = None
self.con_matrix: np.ndarray | None = None
self.unavailable_model_ids: set[str] = set()
def ensure_loaded(self) -> None:
with self._lock:
if self._loaded:
return
cap_index = faiss.read_index(str(CAP_INDEX_FILE))
con_index = faiss.read_index(str(CON_INDEX_FILE))
with METADATA_FILE.open("rb") as handle:
metadata = pickle.load(handle)
from sgar_mvp.src.model_selection import require_registered_models
require_registered_models(metadata["id_to_resource"].values())
indexed_profile = metadata.get("profile_version")
if indexed_profile != PROFILE_VERSION:
raise RuntimeError(
"Retrieval index is stale: "
f"profile={indexed_profile!r}, expected={PROFILE_VERSION!r}. "
"Run build_index.py in the sgar environment."
)
dimension = int(metadata["dim"])
for index in (cap_index, con_index):
index_dimension = getattr(index, "d", dimension)
if int(index_dimension) != dimension:
raise RetrievalLifecycleError("retrieval_index_dimension_mismatch")
cap_matrix = np.asarray(metadata["cap_vectors"], dtype="float32")
con_matrix = np.asarray(metadata["con_vectors"], dtype="float32")
if (
cap_matrix.ndim != 2
or con_matrix.ndim != 2
or cap_matrix.shape[1] != dimension
or con_matrix.shape[1] != dimension
):
raise RetrievalLifecycleError("retrieval_vector_dimension_mismatch")
index_meta_raw = metadata.get("index_meta_v2")
if index_meta_raw:
try:
index_meta = IndexMetaV2.model_validate(index_meta_raw)
except ValueError as exc:
raise RetrievalLifecycleError("retrieval_index_meta_v2_invalid") from exc
resource_order = tuple(
str(metadata["idx_to_id"][index])
for index in range(len(metadata["idx_to_id"]))
)
if resource_order != index_meta.resource_id_order:
raise RetrievalLifecycleError("retrieval_index_resource_order_mismatch")
if hashlib.sha256(
np.ascontiguousarray(cap_matrix, dtype="float32").tobytes()
).hexdigest() != index_meta.capability_vectors_sha256:
raise RetrievalLifecycleError("retrieval_capability_vector_hash_mismatch")
if hashlib.sha256(
np.ascontiguousarray(con_matrix, dtype="float32").tobytes()
).hexdigest() != index_meta.constraint_vectors_sha256:
raise RetrievalLifecycleError("retrieval_constraint_vector_hash_mismatch")
self.cap_index = cap_index
self.con_index = con_index
self.metadata = metadata
self.cap_matrix = cap_matrix
self.con_matrix = con_matrix
self.unavailable_model_ids = _load_unavailable_model_ids()
self._loaded = True
def close(self) -> None:
with self._lock:
self.cap_index = None
self.con_index = None
self.metadata = None
self.cap_matrix = None
self.con_matrix = None
self.unavailable_model_ids = set()
self._loaded = False
class LocalEmbeddingRuntime:
"""Offline local encoder lifecycle independent from index and transport."""
def __init__(
self,
index_runtime: IndexMetadataRuntime,
*,
encoder_factory: Callable[..., Any] | None = None,
) -> None:
self._index_runtime = index_runtime
self._encoder_factory = encoder_factory
self._lock = threading.RLock()
self.encoder: Any | None = None
self.identity: LocalEmbeddingIdentity | None = None
self._snapshot_path: Path | None = None
self.config: EmbeddingRuntimeConfigV1 | None = None
self._runtime: LocalEmbeddingEncoder | None = None
def ensure_config(self) -> EmbeddingRuntimeConfigV1:
with self._lock:
if self.config is not None:
return self.config
self._index_runtime.ensure_loaded()
assert self._index_runtime.metadata is not None
self.config = embedding_config_from_metadata(
self._index_runtime.metadata,
fallback_model_id=EMBED_MODEL,
fallback_query_prefix=BGE_PREFIX,
)
return self.config
def ensure_identity(self) -> LocalEmbeddingIdentity:
with self._lock:
if self.identity is not None:
return self.identity
config = self.ensure_config()
runtime = LocalEmbeddingEncoder(
config,
encoder_factory=self._encoder_factory,
)
identity = runtime.identity()
self._runtime = runtime
self.identity = identity
return identity
def ensure_loaded(self) -> Any:
with self._lock:
if self.encoder is not None:
return self.encoder
identity = self.ensure_identity()
assert self._runtime is not None
encoder = self._runtime.load()
if self.config is None or self.config.output_dimension != identity.index_dimension:
raise RetrievalLifecycleError("embedding_index_dimension_mismatch")
self.encoder = encoder
return encoder
def encode_queries(self, texts: Sequence[str], *, role: str) -> np.ndarray:
self.ensure_loaded()
assert self._runtime is not None
return self._runtime.encode_queries(
texts,
role=role, # type: ignore[arg-type]
batch_size=32,
)
def assert_profile_texts_embeddable(
self, capability_text: str, constraint_text: str
) -> None:
self.ensure_loaded()
assert self._runtime is not None
assert self.config is not None
prepared = (
prepare_query_texts([capability_text], self.config, role="capability")
+ prepare_query_texts([constraint_text], self.config, role="constraint")
)
self._runtime.assert_token_lengths(prepared)
def close(self) -> None:
with self._lock:
self.encoder = None
self.identity = None
self._snapshot_path = None
self.config = None
if self._runtime is not None:
self._runtime.close()
self._runtime = None
class HyDETransportRuntime:
"""Paid HyDE transport lifecycle independent from local retrieval state."""
def __init__(self, *, client_factory: Callable[..., Any] | None = None) -> None:
self._client_factory = client_factory
self._lock = threading.RLock()
self._initialized = False
self.llm_client: Any | None = None
def ensure_loaded(self) -> Any | None:
with self._lock:
if self.llm_client is not None:
return self.llm_client
if self._initialized:
return None
key = _load_env_value("LLM_API_KEY")
if key:
base_url = (
_load_env_value("LLM_BASE_URL", "OPENAI_BASE_URL") or HYDE_BASE_URL
).rstrip("/")
factory = self._client_factory
if factory is None:
from openai import OpenAI
from sgar_mvp.src.direct_network import direct_sync_http_client
from functools import partial
factory = partial(OpenAI, http_client=direct_sync_http_client())
self.llm_client = factory(
api_key=key,
base_url=base_url,
max_retries=0,
)
self._initialized = True
return self.llm_client
def close(self) -> None:
with self._lock:
client = self.llm_client
self.llm_client = None
self._initialized = False
close = getattr(client, "close", None)
if callable(close):
close()
class _RetrievalEngine:
"""Compatibility facade over three explicit, re-entrant runtimes."""
def __init__(
self,
*,
encoder_factory: Callable[..., Any] | None = None,
client_factory: Callable[..., Any] | None = None,
) -> None:
self.index_runtime = IndexMetadataRuntime()
self.embedding_runtime = LocalEmbeddingRuntime(
self.index_runtime,
encoder_factory=encoder_factory,
)
self.hyde_runtime = HyDETransportRuntime(client_factory=client_factory)
def ensure_loaded(self) -> None:
"""Legacy name now means index-only loading."""
self.index_runtime.ensure_loaded()
def ensure_encoder_loaded(self) -> Any:
return self.embedding_runtime.ensure_loaded()
def ensure_transport_loaded(self) -> Any | None:
return self.hyde_runtime.ensure_loaded()
def local_embedding_identity(self) -> LocalEmbeddingIdentity:
return self.embedding_runtime.ensure_identity()
def close_index(self) -> None:
self.index_runtime.close()
def close_encoder(self) -> None:
self.embedding_runtime.close()
def close_transport(self) -> None:
self.hyde_runtime.close()
def close(self) -> None:
self.close_transport()
self.close_encoder()
self.close_index()
@property
def cap_index(self) -> faiss.Index | None:
return self.index_runtime.cap_index
@property
def con_index(self) -> faiss.Index | None:
return self.index_runtime.con_index
@property
def metadata(self) -> Dict[str, Any] | None:
return self.index_runtime.metadata
@property
def cap_matrix(self) -> np.ndarray | None:
return self.index_runtime.cap_matrix
@property
def con_matrix(self) -> np.ndarray | None:
return self.index_runtime.con_matrix
@property
def unavailable_model_ids(self) -> set[str]:
return self.index_runtime.unavailable_model_ids
@property
def encoder(self) -> Any | None:
return self.embedding_runtime.encoder
@encoder.setter
def encoder(self, value: Any | None) -> None:
self.embedding_runtime.encoder = value
@property
def llm_client(self) -> Any | None:
return self.hyde_runtime.llm_client
@llm_client.setter
def llm_client(self, value: Any | None) -> None:
self.hyde_runtime.llm_client = value
self.hyde_runtime._initialized = value is not None
_engine = _RetrievalEngine()
def _encode(text: str, *, role: str = "capability") -> np.ndarray:
_engine.ensure_loaded()
return _engine.embedding_runtime.encode_queries([text], role=role)
def _encode_many(
texts: Sequence[str],
*,
role: str = "capability",
) -> np.ndarray:
_engine.ensure_loaded()
if not texts:
return np.empty((0, get_embedding_dim()), dtype="float32")
return _engine.embedding_runtime.encode_queries(texts, role=role)
def encode_query(text: str, *, role: str = "capability") -> List[float]:
return _encode(text, role=role)[0].tolist()
def infer_hard_requirements(query: str) -> Dict[str, Any]:
"""Extract only explicit, deterministic task gates from the original query."""
lowered = query.lower()
required_features: List[str] = []
input_modalities: List[str] = ["text"]
output_artifacts: List[str] = []
if any(term in lowered for term in ("image", "screenshot", "photo", '\u56fe\u7247', '\u56fe\u50cf', '\u622a\u56fe')):
input_modalities.append("image")
required_features.append("vision")
if any(term in lowered for term in ("tool calling", "function calling", '\u5de5\u5177\u8c03\u7528', '\u51fd\u6570\u8c03\u7528')):
required_features.append("tool_calling")
if any(term in lowered for term in ("json mode", "strict json", "json schema", '\u7ed3\u6784\u5316 json')):
required_features.append("json_mode")
artifact_terms = {
"json": ("json",),
"csv": ("csv",),
"markdown": ("markdown", ".md"),
"code": ("code", '\u4ee3\u7801', '\u6e90\u7801'),
"pdf": ("pdf",),
"image": ("image", '\u56fe\u7247', '\u56fe\u50cf'),
}
for artifact, terms in artifact_terms.items():
if any(term in lowered for term in terms):
output_artifacts.append(artifact)
min_context_tokens: int | None = None
context_match = re.search(
'(\\d+(?:\\.\\d+)?)\\s*([km]?)\\s*(?:tokens?|\u4e0a\u4e0b\u6587|context)',
lowered,
)
if context_match:
value = float(context_match.group(1))
if context_match.group(2) == "k":
value *= 1_000
elif context_match.group(2) == "m":
value *= 1_000_000
min_context_tokens = int(value)
return {
"required_model_features": sorted(set(required_features)),
"input_modalities": sorted(set(input_modalities)),
"output_artifacts": sorted(set(output_artifacts)),
"min_context_tokens": min_context_tokens,
"original_query": query,
}
def _parse_profiler_input(
query: str,
*,
require_typed_input: bool,
) -> Mapping[str, Any]:
try:
profiler_input = strict_json_loads(query)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
if require_typed_input:
raise HyDEGenerationError(
"profiler_input_envelope_invalid",
responsibility="framework",
response_received=False,
) from exc
return {
"protocol": "sgar-profiler-input-compat-v1",
"subtask_contract_text": query,
"source_content_policy": "preserve_original",
"internal_output_language": "en",
}
if not isinstance(profiler_input, Mapping):
if require_typed_input:
raise HyDEGenerationError(
"profiler_input_envelope_invalid",
responsibility="framework",
response_received=False,
)
return {
"protocol": "sgar-profiler-input-compat-v1",
"subtask_contract": profiler_input,
"source_content_policy": "preserve_original",
"internal_output_language": "en",
}
if require_typed_input:
if profiler_input.get("protocol") != "sgar-profiler-input-v1":
raise HyDEGenerationError(
"profiler_input_envelope_protocol_invalid",
responsibility="framework",
response_received=False,
)
try:
return ProfilerInputEnvelopeV1.model_validate(profiler_input).model_dump(
mode="json"
)
except ValueError as exc:
raise HyDEGenerationError(
"profiler_input_envelope_invalid",
responsibility="framework",
response_received=False,
) from exc
return profiler_input
def _generate_hyde_result(
query: str,
*,
allow_direct_fallback: bool | None = None,
cost_ledger: RunCostLedger | None = None,
subtask_id: str | None = None,
subtask_revision: int | None = None,
transport: SyncModelTransportPort | None = None,
response_mode: StructuredResponseModeInput = "native_strict_schema",
model_id: str | None = None,
reasoning_effort: str | None = None,
require_typed_input: bool = False,
prompt_version: str | None = None,
system_prompt: str | None = None,
max_output_tokens: int | None = None,
correction_diagnostics: Mapping[str, Any] | None = None,
model_attempt_limit: int | None = None,
source_seal_path: Path | None = None,
) -> Dict[str, Any]:
"""Generate one strict, pool-blind Profiler profile with bounded correction."""
selected_seal_path = source_seal_path
if selected_seal_path is None and os.environ.get("SGAR_SOURCE_SEAL_PATH"):
selected_seal_path = Path(os.environ["SGAR_SOURCE_SEAL_PATH"])
source_seal = (
load_and_verify_source_seal(
selected_seal_path,
project_root=PROJECT_ROOT,
allowed_stages=("release", "activated"),
)
if selected_seal_path is not None
else None
)
profiler_input = _parse_profiler_input(
query,
require_typed_input=require_typed_input,
)
if transport is None:
_engine.ensure_transport_loaded()
fallback_allowed = (
RETRIEVAL_POLICY.hyde.allow_direct_fallback
if allow_direct_fallback is None
else allow_direct_fallback
)
if transport is None and _engine.llm_client is None:
if fallback_allowed:
return {
"capability_text": query,
"constraint_text": query,
"think": "Direct fallback used because no Profiler client was available.",
"metadata": {"status": "direct_fallback", "reason": "missing_api_key"},
}
raise HyDEGenerationError(
"hyde_client_unavailable",
responsibility="framework",
response_received=False,
)
if transport is None:
transport = get_profiler_transport()
transport = require_sync_model_transport(transport)
started = time.perf_counter()
profiler_requirement = system_role_requirement("hyde")
selected_mode = normalize_structured_response_mode(response_mode)
selected_model = str(model_id or HYDE_MODEL).strip()
if not selected_model:
raise HyDEGenerationError(
"profiler_model_id_missing",
responsibility="framework",
response_received=False,
)
selected_reasoning_effort = str(
reasoning_effort or RETRIEVAL_POLICY.hyde.reasoning_effort
).strip()
selected_prompt_version = str(prompt_version or HYDE_PROFILE_VERSION).strip()
expected_system_prompt = _profiler_prompt_for_version(selected_prompt_version)
selected_system_prompt = str(system_prompt or expected_system_prompt)
if selected_system_prompt != expected_system_prompt:
raise HyDEGenerationError(
"profiler_prompt_version_content_mismatch",
responsibility="framework",
response_received=False,
)
if selected_model != RETRIEVAL_POLICY.hyde.api_model_id:
raise HyDEGenerationError(
"profiler_model_policy_violation",
responsibility="framework",
response_received=False,
)
if selected_reasoning_effort != RETRIEVAL_POLICY.hyde.reasoning_effort:
raise HyDEGenerationError(
"profiler_reasoning_effort_policy_violation",
responsibility="framework",
response_received=False,
)
accounting_context = (
cost_ledger.new_operation(
stage="retrieval_hyde",
subtask_id=subtask_id,
subtask_revision=subtask_revision,
request_policy_sha256=RETRIEVAL_POLICY.sha256(),
reasoning_effort=selected_reasoning_effort,
)
if cost_ledger is not None
else None
)
selected_max_output_tokens = int(
max_output_tokens or RETRIEVAL_POLICY.hyde.max_tokens
)
if selected_max_output_tokens <= 0:
raise HyDEGenerationError(
"profiler_max_output_tokens_invalid",
responsibility="framework",
response_received=False,
)
selected_model_attempt_limit = int(
model_attempt_limit or RETRIEVAL_POLICY.hyde.max_model_attempts
)
if selected_model_attempt_limit not in {1, 2}:
raise HyDEGenerationError(
"profiler_model_attempt_limit_invalid",
responsibility="framework",
response_received=False,
)
request_hashes: List[str] = []
model_request_hashes: List[str] = []
model_attempt_output_caps: List[int] = []
response: Any | None = None
last_failure_code = "profiler_response_invalid"
last_request_sha256: str | None = None
def build_messages(
*,
correction_code: str | None,
) -> list[dict[str, str]]:
user_payload: Dict[str, Any]
if selected_mode == "json_object_local_validator":
user_payload = {
"profiler_input": profiler_input,
"authoritative_output_schema": profiler_requirement.json_schema,
}
else:
user_payload = dict(profiler_input)
messages = [
{"role": "system", "content": selected_system_prompt},
{
"role": "user",
"content": json.dumps(
user_payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
),
},
]
if correction_code is not None:
diagnostic_payload = dict(correction_diagnostics or {})
messages.append(
{
"role": "user",
"content": json.dumps(
{
"protocol": "sgar-profiler-correction-v1",
"failure_code": correction_code,
"diagnostics": diagnostic_payload,
"instruction": (
"Return a complete replacement object matching the same "
"Profiler schema. Preserve the subtask semantics and correct "
"only the reported output-contract failure."
),
},
sort_keys=True,
separators=(",", ":"),
),
}
)
return messages
try:
for model_attempt in range(1, selected_model_attempt_limit + 1):
attempt_output_cap = (
int(RETRIEVAL_POLICY.hyde.truncation_retry_max_output_tokens)
if model_attempt == 2
and last_failure_code == "profiler_output_truncated"
and selected_max_output_tokens < int(
RETRIEVAL_POLICY.hyde.truncation_retry_max_output_tokens
)
else selected_max_output_tokens
)
model_attempt_output_caps.append(attempt_output_cap)
request_payload = {
"model": selected_model,
"messages": build_messages(
correction_code=(
"profiler_profile_revision_required"
if correction_diagnostics is not None and model_attempt == 1
else last_failure_code if model_attempt == 2 else None
)
),
"max_tokens": attempt_output_cap,
"response_format": structured_response_format(
profiler_requirement,
mode=selected_mode,
),
}
if RETRIEVAL_POLICY.hyde.temperature is not None:
request_payload["temperature"] = RETRIEVAL_POLICY.hyde.temperature
if selected_reasoning_effort:
request_payload["reasoning_effort"] = selected_reasoning_effort
expected_request_sha256 = model_request_sha256(request_payload)
last_request_sha256 = expected_request_sha256
model_request_hashes.append(expected_request_sha256)
response = None
for transport_attempt in range(
1,
RETRIEVAL_POLICY.hyde.max_transport_attempts_per_model_attempt + 1,
):
attempt_payload = deepcopy(request_payload)
attempt_hash = model_request_sha256(attempt_payload)
if attempt_hash != expected_request_sha256:
raise HyDEGenerationError(
"profiler_transport_request_changed",
responsibility="framework",
response_received=False,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
)
request_hashes.append(attempt_hash)
try:
response = transport.send(
ledger=cost_ledger,
context=accounting_context,
**attempt_payload,
)
break
except BudgetControlError as exc:
raise HyDEGenerationError(
str(exc.error_code),
responsibility="budget",
response_received=False,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
) from exc
except (ModelAccountingError, ModelTransportError) as exc:
raise HyDEGenerationError(
type(exc).__name__,
responsibility="framework",
response_received=False,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
) from exc
except Exception as exc:
retryable, failure_code = classify_transport_exception(exc)
if (
retryable
and transport_attempt
< RETRIEVAL_POLICY.hyde.max_transport_attempts_per_model_attempt
and not (cost_ledger is not None and cost_ledger.is_exhausted)
and wait_for_proxy_retry(exc, transport_attempt)
):
continue
raise HyDEGenerationError(
failure_code,
responsibility="infrastructure" if retryable else "framework",
response_received=False,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
) from exc
if response is None:
raise HyDEGenerationError(
"profiler_transport_no_terminal_response",
responsibility="framework",
response_received=False,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
)
choice = response.choices[0]
finish_reason = str(getattr(choice, "finish_reason", "") or "")
full_text = str(choice.message.content or "").strip()
failure_code: str | None = None
if finish_reason == "length":
failure_code = "profiler_output_truncated"
else:
try:
parsed = strict_json_loads(full_text)
except (TypeError, ValueError, json.JSONDecodeError):
parsed = None
failure_code = "profiler_response_json_invalid"
if failure_code is None and not isinstance(parsed, Mapping):
failure_code = "profiler_response_schema_invalid"
if failure_code is None and selected_mode == "native_strict_schema":
parsed = normalize_portable_wire_instance(
parsed,
profiler_requirement.json_schema or {},
)
if failure_code is None:
valid, _reason = validate_json_schema_instance(
parsed,
profiler_requirement.json_schema or {},
)
if not valid:
failure_code = "profiler_response_schema_invalid"
if failure_code is None:
try:
output = ProfilerOutputV2.model_validate(parsed)
except ValueError:
failure_code = "profiler_response_fields_empty"
if (
failure_code is None
and not _profiler_output_is_english(output, query)
):
failure_code = "profiler_output_language_invalid"
if (
failure_code is None
and isinstance(output, ProfilerOutputV2)
and not _profiler_think_is_valid(output)
):
failure_code = "profiler_think_invalid"
if (
failure_code is None
and isinstance(output, ProfilerOutputV2)
and not _profiler_output_is_safe(output, query)
):
failure_code = "profiler_output_leakage_invalid"
if failure_code is not None:
last_failure_code = failure_code
if model_attempt < selected_model_attempt_limit:
continue
raise HyDEGenerationError(
failure_code,
responsibility="research",
response_received=True,
transport_attempts=len(request_hashes),
request_sha256=expected_request_sha256,
)
usage = getattr(response, "usage", None)
completion_details = getattr(usage, "completion_tokens_details", None)
if isinstance(completion_details, Mapping):
reasoning_tokens = int(completion_details.get("reasoning_tokens") or 0)
else:
reasoning_tokens = int(
getattr(completion_details, "reasoning_tokens", 0) or 0
)
usage_payload = {
"prompt_tokens": int(getattr(usage, "prompt_tokens", 0) or 0),
"completion_tokens": int(getattr(usage, "completion_tokens", 0) or 0),
"reasoning_tokens": reasoning_tokens,
"total_tokens": int(getattr(usage, "total_tokens", 0) or 0),
}
reasoning_observation = observe_provider_reasoning(response)
result = {
"capability_text": output.capability_text.strip(),
"constraint_text": output.constraint_text.strip(),
"metadata": {
"status": "generated",
"model_id": selected_model,
"reasoning_effort": selected_reasoning_effort,
"effective_max_output_tokens": attempt_output_cap,
"normal_max_output_tokens": selected_max_output_tokens,
"truncation_retry_max_output_tokens": int(
RETRIEVAL_POLICY.hyde.truncation_retry_max_output_tokens
),
"model_attempt_output_caps": list(model_attempt_output_caps),
"truncation_retry_used": model_attempt_output_caps == [
selected_max_output_tokens,
int(RETRIEVAL_POLICY.hyde.truncation_retry_max_output_tokens),
],
"prompt_version": selected_prompt_version,
"prompt_sha256": hashlib.sha256(
selected_system_prompt.encode("utf-8")
).hexdigest(),
"latency_ms": round((time.perf_counter() - started) * 1000, 3),
"usage": usage_payload,
"model_attempt_count": model_attempt,
"model_attempt_limit": selected_model_attempt_limit,
"transport_attempt_count": len(request_hashes),
"transport_request_sha256": expected_request_sha256,
"transport_request_hashes": list(request_hashes),
"model_request_hashes": list(model_request_hashes),
"response_sha256": hashlib.sha256(
full_text.encode("utf-8")
).hexdigest(),
"finish_reason": finish_reason,
"response_mode": selected_mode,
"endpoint_identity_sha256": transport.endpoint_identity.identity_sha256,
"wire_schema_sha256": profiler_requirement.wire_schema_sha256,
"provider_reasoning_observation": reasoning_observation.model_dump(
mode="json"
),
"model_accounting_reference": getattr(
response,
"accounting_reference",
None,
),
"source_seal": (
source_seal_reference(source_seal)
if source_seal is not None
else None
),
},
}
result["think"] = output.think.strip()
return result
raise HyDEGenerationError(
last_failure_code,
responsibility="research",
response_received=True,
transport_attempts=len(request_hashes),
request_sha256=last_request_sha256,
)
except HyDEGenerationError as exc:
if fallback_allowed:
logging.getLogger(__name__).warning(
"HyDE failed; explicit direct fallback: %s",
exc.failure_code,
)
return {
"capability_text": query,
"constraint_text": query,
"think": "Direct fallback used after the Profiler request failed.",
"metadata": {
"status": "direct_fallback",
"reason": exc.failure_code,
"latency_ms": round((time.perf_counter() - started) * 1000, 3),
"transport_attempt_count": exc.transport_attempts,
"transport_request_sha256": exc.request_sha256,
},
}
raise
except Exception as exc:
if fallback_allowed:
logging.getLogger(__name__).warning(
"HyDE failed; explicit direct fallback: %s",
type(exc).__name__,
)
return {
"capability_text": query,
"constraint_text": query,
"think": "Direct fallback used after Profiler response processing failed.",
"metadata": {
"status": "direct_fallback",
"reason": type(exc).__name__,
"latency_ms": round((time.perf_counter() - started) * 1000, 3),
},
}
raise HyDEGenerationError(
"profiler_response_processing_failed",
responsibility="research" if response is not None else "framework",
response_received=response is not None,
transport_attempts=len(request_hashes),
request_sha256=last_request_sha256,
) from exc
def generate_hyde_profile_text(
query: str,
*,
cost_ledger: RunCostLedger | None = None,
subtask_id: str | None = None,
subtask_revision: int | None = None,
transport: SyncModelTransportPort | None = None,
response_mode: StructuredResponseModeInput = "native_strict_schema",
model_id: str | None = None,
reasoning_effort: str | None = None,
require_typed_input: bool = False,
prompt_version: str | None = None,
system_prompt: str | None = None,
max_output_tokens: int | None = None,
correction_diagnostics: Mapping[str, Any] | None = None,
model_attempt_limit: int | None = None,
source_seal_path: Path | None = None,
) -> Dict[str, Any]:
"""Generate exactly one semantic ideal-profile response.
Provider transport retries are internal to that one semantic operation.
Local embedding/index retries must call ``encode_stored_query_profiles``
with the returned texts and must never call this function again.
"""
return _generate_hyde_result(
query,
allow_direct_fallback=False,
cost_ledger=cost_ledger,
subtask_id=subtask_id,
subtask_revision=subtask_revision,
transport=transport,
response_mode=response_mode,
model_id=model_id,
reasoning_effort=reasoning_effort,
require_typed_input=require_typed_input,
prompt_version=prompt_version,
system_prompt=system_prompt,
max_output_tokens=max_output_tokens,
correction_diagnostics=correction_diagnostics,
model_attempt_limit=model_attempt_limit,
source_seal_path=source_seal_path,
)
def profiler_output_is_embeddable(
output: ProfilerOutputV2,
) -> bool:
"""Return false only for an explicit tokenizer-limit violation."""
try:
_engine.embedding_runtime.assert_profile_texts_embeddable(
output.capability_text, output.constraint_text
)
except RetrievalLifecycleError as exc:
if str(exc.failure_code).startswith("embedding_input_token_limit_exceeded"):
return False
raise
return True
def generate_hyde(
query: str,
*,
cost_ledger: RunCostLedger | None = None,
transport: SyncModelTransportPort | None = None,
) -> Tuple[str, str]:
result = _generate_hyde_result(
query,
cost_ledger=cost_ledger,
transport=transport,
)
return str(result["capability_text"]), str(result["constraint_text"])
def encode_query_profile(
query: str,
use_hyde: bool = True,
*,
cost_ledger: RunCostLedger | None = None,
subtask_id: str | None = None,
subtask_revision: int | None = None,
) -> QueryProfile:
if use_hyde:
result = _generate_hyde_result(
query,
cost_ledger=cost_ledger,
subtask_id=subtask_id,
subtask_revision=subtask_revision,
)
capability_text = str(result["capability_text"])
constraint_text = str(result["constraint_text"])
generation_metadata = dict(result["metadata"])
else:
capability_text = query
constraint_text = query
generation_metadata = {"status": "no_hyde"}
return QueryProfile(
capability_text=capability_text,
constraint_text=constraint_text,
capability_vector=encode_query(capability_text, role="capability"),
constraint_vector=encode_query(constraint_text, role="constraint"),
raw_query_text=query,
raw_query_vector=encode_query(query, role="raw"),
hard_requirements=infer_hard_requirements(query),
profile_version=HYDE_PROFILE_VERSION if use_hyde else "direct-query-v1",
generation_metadata=generation_metadata,
)
def encode_query_profiles(
queries: Sequence[str],
use_hyde: bool = True,
hyde_workers: int | None = None,
*,
cost_ledger: RunCostLedger | None = None,
) -> List[QueryProfile]:
"""Batch-encode profiles; HyDE generation remains one request per query."""
if use_hyde:
_engine.ensure_loaded()
worker_count = hyde_workers or int(os.environ.get("SGAR_HYDE_WORKERS", "6"))
with ThreadPoolExecutor(max_workers=max(1, min(worker_count, 12))) as executor:
generated = list(
executor.map(
lambda item: _generate_hyde_result(
item,
cost_ledger=cost_ledger,
),
queries,
)
)
else:
generated = [
{
"capability_text": query,
"constraint_text": query,
"metadata": {"status": "no_hyde"},
}
for query in queries
]
text_pairs = [
(str(item["capability_text"]), str(item["constraint_text"]))
for item in generated
]
capability_texts = [pair[0] for pair in text_pairs]
constraint_texts = [pair[1] for pair in text_pairs]
raw_query_vectors = _encode_many(list(queries), role="raw")
if use_hyde:
cap_vectors = _encode_many(capability_texts, role="capability")
con_vectors = _encode_many(constraint_texts, role="constraint")
else:
# Direct mode has identical texts; encode once for deterministic CI.
cap_vectors = _encode_many(capability_texts, role="capability")
con_vectors = cap_vectors
return [
QueryProfile(
capability_text=capability_text,
constraint_text=constraint_text,
capability_vector=cap_vectors[index].tolist(),
constraint_vector=con_vectors[index].tolist(),
raw_query_text=queries[index],
raw_query_vector=raw_query_vectors[index].tolist(),
hard_requirements=infer_hard_requirements(queries[index]),
profile_version=HYDE_PROFILE_VERSION if use_hyde else "direct-query-v1",
generation_metadata=dict(generated[index]["metadata"]),
)
for index, (capability_text, constraint_text) in enumerate(text_pairs)
]
def encode_stored_query_profiles(
stored_profiles: Sequence[Mapping[str, Any]],
) -> List[QueryProfile]:
"""Re-encode cached HyDE texts without making provider/API calls."""
capability_texts = [str(item.get("capability_text") or "") for item in stored_profiles]
constraint_texts = [str(item.get("constraint_text") or "") for item in stored_profiles]
if any(not text for text in capability_texts + constraint_texts):
raise ValueError("Stored query profile is missing capability/constraint text")
cap_vectors = _encode_many(capability_texts, role="capability")
con_vectors = _encode_many(constraint_texts, role="constraint")
raw_query_texts = [
str(
(item.get("hard_requirements") or {}).get("original_query")
or item.get("query")
or ""
)
for item in stored_profiles
]
if any(not text for text in raw_query_texts):
raise ValueError("Stored query profile is missing its original query")
raw_vectors = _encode_many(raw_query_texts, role="raw")
return [
QueryProfile(
capability_text=capability_texts[index],
constraint_text=constraint_texts[index],
capability_vector=cap_vectors[index].tolist(),
constraint_vector=con_vectors[index].tolist(),
raw_query_text=raw_query_texts[index],
raw_query_vector=raw_vectors[index].tolist(),
hard_requirements=dict(item.get("hard_requirements") or {}),
profile_version=str(item.get("profile_version") or "typed-query-v1"),
generation_metadata=dict(item.get("generation_metadata") or {"status": "cache"}),
)
for index, item in enumerate(stored_profiles)
]
def attach_concatenated_vectors(
profiles: Sequence[QueryProfile],
) -> List[QueryProfile]:
"""Batch-compute the legacy concatenated-HyDE vector for ablation only."""
missing = [profile for profile in profiles if profile.concatenated_vector is None]
if not missing:
return list(profiles)
combined = _encode_many(
[
profile.capability_text + "\n---\n" + profile.constraint_text
for profile in missing
]
)
iterator = iter(combined)
output: List[QueryProfile] = []
for profile in profiles:
if profile.concatenated_vector is None:
output.append(replace(profile, concatenated_vector=next(iterator).tolist()))
else:
output.append(profile)
return output
def _model_api_id(raw: Mapping[str, Any]) -> str:
type_specific = raw.get("type_specific")
model = type_specific.get("model") if isinstance(type_specific, Mapping) else {}
execution = raw.get("execution")
execution = execution if isinstance(execution, Mapping) else {}
return str(
(model.get("model_id") if isinstance(model, Mapping) else None)
or execution.get("model_id")
or ""
)
def resource_passes_hard_requirements(
resource_id: str,
task_requirements: Mapping[str, Any] | None = None,
) -> Tuple[bool, str | None]:
"""Apply deterministic base and explicit task gates."""
_engine.ensure_loaded()
assert _engine.metadata is not None
raw = _engine.metadata["id_to_resource"].get(resource_id)
if not isinstance(raw, Mapping):
return False, "resource_not_indexed"
from sgar_mvp.src.model_selection import is_candidate_resource
if not is_candidate_resource(raw):
return False, "model_not_candidate"
hard = _engine.metadata.get("hard_requirements", {}).get(resource_id, {})
status = str(hard.get("status") or "active").lower()
execution_status = str(hard.get("execution_status") or "active").lower()
if status not in {"active", "available", "ok", "ready", ""}:
return False, "inactive_resource"
if execution_status not in {"active", "available", "ok", "ready", ""}:
return False, "inactive_execution"
rtype = _resource_type(raw)
if rtype == "Model":
model_id = _model_api_id(raw)
if resource_id in _engine.unavailable_model_ids or model_id in _engine.unavailable_model_ids:
return False, "model_health_gate"
requirements = task_requirements or {}
supports = hard.get("supports") if isinstance(hard.get("supports"), Mapping) else {}
for feature in requirements.get("required_model_features", []):
if supports.get(feature) is not True:
return False, f"missing_model_feature:{feature}"
minimum = requirements.get("min_context_tokens")
available = hard.get("context_tokens")
if minimum and (not available or int(available) < int(minimum)):
return False, "insufficient_context_window"
if rtype == "Skill":
requirements = task_requirements or {}
query = str(requirements.get("original_query") or "").lower()
for cue in hard.get("avoid_when", []):
normalized = str(cue).strip().lower()
if len(normalized) >= 5 and normalized in query:
return False, "skill_avoid_when"
ids = set(_engine.metadata["id_to_resource"])
missing = set(hard.get("required_resource_ids", [])) - ids
if missing:
return False, "missing_skill_dependency"
return True, None
def _semantic_scores(
profile: QueryProfile,
strategy: str,
resource_type: str | None,
type_weights: Mapping[str, Tuple[float, float]] | None = None,
) -> List[Dict[str, Any]]:
_engine.ensure_loaded()
assert _engine.metadata is not None
assert _engine.cap_matrix is not None and _engine.con_matrix is not None
if strategy == "concatenated_hyde":
combined = profile.concatenated_vector or encode_query(
profile.capability_text + "\n---\n" + profile.constraint_text
)
cap_query = np.asarray(combined, dtype="float32")
con_query = cap_query
else:
cap_query = np.asarray(profile.capability_vector, dtype="float32")
con_query = np.asarray(profile.constraint_vector, dtype="float32")
cap_scores = _engine.cap_matrix @ cap_query
con_scores = _engine.con_matrix @ con_query
raw_scores: np.ndarray | None = None
if strategy == "raw_capability_rrf":
if profile.raw_query_vector is None:
raise ValueError("raw_capability_rrf requires an encoded raw query")
raw_query = np.asarray(profile.raw_query_vector, dtype="float32")
raw_scores = _engine.cap_matrix @ raw_query
# RRF is computed independently inside each resource type. This prevents
# the score distribution of a large Tool pool from displacing Model,
# Agent, or Skill candidates and preserves the typed retrieval contract.
rrf_scores: Dict[int, float] = {}
if raw_scores is not None:
grouped: Dict[str, List[int]] = {}
for index in range(len(cap_scores)):
resource_id = _engine.metadata["idx_to_id"][index]
raw = _engine.metadata["id_to_resource"][resource_id]
rtype = _resource_type(raw)
from sgar_mvp.src.model_selection import is_candidate_resource
if not is_candidate_resource(raw):
continue
if resource_type and rtype != resource_type:
continue
grouped.setdefault(rtype, []).append(index)
rrf_k = 60.0
normalizer = 2.0 / (rrf_k + 1.0)
for indices in grouped.values():
cap_order = sorted(
indices,
key=lambda item: (float(cap_scores[item]), _engine.metadata["idx_to_id"][item]),
reverse=True,
)
raw_order = sorted(
indices,
key=lambda item: (float(raw_scores[item]), _engine.metadata["idx_to_id"][item]),
reverse=True,
)
cap_rank = {item: rank for rank, item in enumerate(cap_order, start=1)}
raw_rank = {item: rank for rank, item in enumerate(raw_order, start=1)}
for item in indices:
rrf_scores[item] = (
1.0 / (rrf_k + cap_rank[item])
+ 1.0 / (rrf_k + raw_rank[item])
) / normalizer
output: List[Dict[str, Any]] = []
for index in range(len(cap_scores)):
resource_id = _engine.metadata["idx_to_id"][index]
raw = _engine.metadata["id_to_resource"][resource_id]
rtype = _resource_type(raw)
from sgar_mvp.src.model_selection import is_candidate_resource
if not is_candidate_resource(raw):
continue
if resource_type and rtype != resource_type:
continue
cap_score = float(cap_scores[index])
con_score = float(con_scores[index])
if strategy == "capability_only":
semantic_score = cap_score
elif strategy == "raw_capability_rrf":
semantic_score = rrf_scores[index]
else:
w_cap, w_con = (type_weights or TYPE_WEIGHTS).get(rtype, (0.70, 0.30))
semantic_score = w_cap * cap_score + w_con * con_score
output.append(
{
"resource_id": resource_id,
"resource_type": rtype,
"score": semantic_score,
"capability_score": cap_score,
"constraint_score": con_score,
"raw_query_score": (
float(raw_scores[index]) if raw_scores is not None else None
),
"utility_adjustment": 0.0,
"hard_gate": "not_applied",
}
)
return output
def _apply_utility_rerank(items: List[Dict[str, Any]]) -> None:
assert _engine.metadata is not None
empirical = [
item
for item in items
if _engine.metadata.get("utility_profiles", {})
.get(item["resource_id"], {})
.get("ranking_enabled")
]
if not empirical:
return
for item in empirical:
utility = _engine.metadata["utility_profiles"][item["resource_id"]]
success = max(0.0, min(1.0, float(utility.get("expected_success_rate") or 0.5)))
# Bounded to ±0.025 so semantic fit remains dominant.
adjustment = 0.05 * (success - 0.5)
item["utility_adjustment"] = adjustment
item["score"] += adjustment
def rank_resources(
profile: QueryProfile,
*,
strategy: str | None = None,
resource_type: str | None = None,
top_k: int = TOP_K,
type_weights: Mapping[str, Tuple[float, float]] | None = None,
) -> List[Dict[str, Any]]:
selected_strategy = str(strategy or RETRIEVAL_POLICY.active_strategy)
return rank_resources_with_trace(
profile,
strategy=selected_strategy,
resource_type=resource_type,
top_k=top_k,
type_weights=type_weights,
)["ranked"]
def rank_resources_with_trace(
profile: QueryProfile,
*,
strategy: str | None = None,
resource_type: str | None = None,
top_k: int = TOP_K,
type_weights: Mapping[str, Tuple[float, float]] | None = None,
) -> Dict[str, List[Dict[str, Any]]]:
strategy = str(strategy or RETRIEVAL_POLICY.active_strategy)
if strategy not in RETRIEVAL_STRATEGIES and strategy != SHADOW_RETRIEVAL_STRATEGY:
raise ValueError(f"Unknown retrieval strategy: {strategy}")
items = _semantic_scores(profile, strategy, resource_type, type_weights)
pre_gate = sorted(
(dict(item) for item in items),
key=lambda item: (item["score"], item["capability_score"]),
reverse=True,
)
rejected: List[Dict[str, Any]] = []
if strategy in {"dual_hard", "dual_hard_utility"}:
gated: List[Dict[str, Any]] = []
for item in items:
passed, reason = resource_passes_hard_requirements(
item["resource_id"], profile.hard_requirements
)
item["hard_gate"] = "passed" if passed else str(reason)
if passed:
gated.append(item)
else:
rejected.append(dict(item))
items = gated
if strategy == "dual_hard_utility":
_apply_utility_rerank(items)
items.sort(key=lambda item: (item["score"], item["capability_score"]), reverse=True)
return {
"ranked": items[: max(0, top_k)],
"pre_gate": pre_gate,
"rejected": rejected,
}
def rank_resources_shadow(
profile: QueryProfile,
*,
resource_type: str | None = None,
top_k: int = TOP_K,
type_weights: Mapping[str, Tuple[float, float]] | None = None,
) -> List[Dict[str, Any]]:
"""Rank with capability/constraint weights without changing active policy.
This is intentionally an explicit audit-only entry point. It shares the
same index, candidate filtering, and per-type weights as formal retrieval,
but cannot be selected through ``retrieval_policy.json``.
"""
return rank_resources_with_trace(
profile,
strategy=SHADOW_RETRIEVAL_STRATEGY,
resource_type=resource_type,
top_k=top_k,
type_weights=type_weights,
)["ranked"]
def utility_ranking_active_count() -> int:
_engine.ensure_loaded()
assert _engine.metadata is not None
return sum(
1
for item in _engine.metadata.get("utility_profiles", {}).values()
if item.get("ranking_enabled")
)
def direct_retrieve(query: str, top_k: int = TOP_K, verbose: bool = True) -> List[Dict[str, Any]]:
profile = encode_query_profile(query, use_hyde=False)
ranked = rank_resources(
profile,
strategy=RETRIEVAL_POLICY.active_strategy,
top_k=top_k,
)
if verbose:
for item in ranked:
print(
f"{item['resource_id']} ({item['resource_type']}) "
f"score={item['score']:.4f} "
f"cap={item['capability_score']:.4f} "
f"con={item['constraint_score']:.4f}"
)
assert _engine.metadata is not None
return [_engine.metadata["id_to_resource"][item["resource_id"]] for item in ranked]
def hyde_retrieve(query: str, top_k: int = TOP_K, verbose: bool = True) -> List[Dict[str, Any]]:
profile = encode_query_profile(query, use_hyde=True)
ranked = rank_resources(
profile,
strategy=RETRIEVAL_POLICY.active_strategy,
top_k=top_k,
)
if verbose:
print(f"Capability HyDE: {profile.capability_text[:160]}")
print(f"Constraint HyDE: {profile.constraint_text[:160]}")
for item in ranked:
print(
f"{item['resource_id']} ({item['resource_type']}) "
f"score={item['score']:.4f} "
f"cap={item['capability_score']:.4f} "
f"con={item['constraint_score']:.4f}"
)
assert _engine.metadata is not None
return [_engine.metadata["id_to_resource"][item["resource_id"]] for item in ranked]
def get_resource_vectors(resource_id: str) -> Tuple[List[float], List[float]]:
_engine.ensure_loaded()
assert _engine.metadata is not None
alias = _engine.metadata.get("duplicate_resource_aliases", {}).get(resource_id, resource_id)
index = _engine.metadata["id_to_idx"][alias]
return (
_engine.metadata["cap_vectors"][index],
_engine.metadata["con_vectors"][index],
)
def get_all_resources() -> List[Dict[str, Any]]:
_engine.ensure_loaded()
assert _engine.metadata is not None
return list(_engine.metadata["id_to_resource"].values())
def get_embedding_dim() -> int:
_engine.ensure_loaded()
assert _engine.metadata is not None
return int(_engine.metadata["dim"])
def get_index_metadata() -> Dict[str, Any]:
_engine.ensure_loaded()
assert _engine.metadata is not None
return _engine.metadata
def get_local_embedding_identity() -> LocalEmbeddingIdentity:
"""Validate and return the host-free offline embedding identity."""
_engine.ensure_loaded()
return _engine.local_embedding_identity()
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("query")
parser.add_argument("--direct", action="store_true")
parser.add_argument("--top-k", type=int, default=TOP_K)
arguments = parser.parse_args()
if arguments.direct:
direct_retrieve(arguments.query, top_k=arguments.top_k)
else:
hyde_retrieve(arguments.query, top_k=arguments.top_k)