ChatDev/execorchestra/build_index.py
2026-11-05 16:22:12 +08:00

567 lines
22 KiB
Python

"""Build the S-GAR dual-vector FAISS index.
``v_cap`` and ``v_con`` are intentionally generated from different textual
views. Hard executability requirements and utility metrics are stored as index
metadata and never embedded into the constraint vector.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import logging
import os
import pickle
import uuid
from pathlib import Path
from typing import Any, Dict, List
import certifi
import faiss
import numpy as np
from sentence_transformers import SentenceTransformer
from sgar_mvp.src.bge_encoding import (
prepare_passage_texts,
)
from sgar_mvp.src.embedding_runtime import (
EmbeddingRuntimeIdentityV2,
EmbeddingRuntimeConfigV1,
IndexMetaV2,
LocalEmbeddingEncoder,
load_embedding_release_config,
)
from sgar_mvp.src.capability_cards import build_capability_consistency_report
from sgar_mvp.src.retrieval_policy import load_retrieval_policy
from sgar_mvp.src.release_environment import require_release_storage_path
from sgar_mvp.src.release_source_seal import (
load_and_verify_source_seal,
source_seal_reference,
)
from retrieval_profiles import (
PROFILE_VERSION,
build_constraint_profile,
deduplicate_exact_skill_packages,
validate_resource_profiles,
)
PROJECT_ROOT = Path(__file__).resolve().parent
CATALOG_FILE = PROJECT_ROOT / "Pool" / "resources" / "json" / "combine.json"
RESOURCES_FILE = PROJECT_ROOT / "Pool" / "resources" / "json" / "effective_combine.json"
# The active pool is the frozen R_exp v1.3 projection. The RC1 readiness
# report is retained for historical maintenance and must not feed a new index.
READINESS_FILE = PROJECT_ROOT / "sgar_mvp" / "config" / "r_exp_v1_3_readiness.json"
INDEX_DIR = PROJECT_ROOT / "Pool" / "index_meta"
CAP_INDEX_FILE = INDEX_DIR / "faiss_cap.index"
CON_INDEX_FILE = INDEX_DIR / "faiss_con.index"
METADATA_FILE = INDEX_DIR / "resource_meta.pkl"
AUDIT_FILE = INDEX_DIR / "retrieval_profile_audit.json"
BUILD_MANIFEST_FILE = INDEX_DIR / "index_build_manifest.json"
EMBEDDING_RELEASE_CONFIG_FILE = (
PROJECT_ROOT / "sgar_mvp" / "config" / "embedding_release.json"
)
_FORMAL_RETRIEVAL_POLICY = load_retrieval_policy()
EMBED_MODEL = _FORMAL_RETRIEVAL_POLICY.embedding_model
os.environ["SSL_CERT_FILE"] = certifi.where()
os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
def load_resources(path: Path = RESOURCES_FILE) -> List[Dict[str, Any]]:
with path.open("r", encoding="utf-8-sig") as handle:
payload = json.load(handle)
if not isinstance(payload, list):
raise ValueError(f"{path} must contain a JSON array")
# Empty legacy aggregate entries are never valid retrievable resources.
resources = [item for item in payload if isinstance(item, dict) and item.get("resource_id")]
from sgar_mvp.src.model_selection import require_registered_models
require_registered_models(resources)
print(f"[build_index] Loaded {len(resources)} valid resources")
return resources
NON_INDEXABLE_STATUSES = {"inactive", "disabled", "unavailable"}
def is_indexable_resource(resource: dict) -> bool:
from sgar_mvp.src.model_selection import is_candidate_resource
if not is_candidate_resource(resource):
return False
status = str(resource.get("status") or "").strip().lower()
execution = resource.get("execution")
if not isinstance(execution, dict):
execution = {}
execution_status = str(execution.get("execution_status") or "").strip().lower()
return bool(resource.get("resource_id")) and not (
status in NON_INDEXABLE_STATUSES
or execution_status in NON_INDEXABLE_STATUSES
)
def filter_indexable_resources(resources: list[dict]) -> list[dict]:
return [resource for resource in resources if is_indexable_resource(resource)]
def resource_to_cap_text(resource: Dict[str, Any]) -> str:
"""Backward-compatible public helper used by existing audits/tests."""
if not isinstance(resource.get("capability"), dict):
return str(resource.get("resource_id") or resource.get("id") or "")
return build_constraint_profile(resource).capability_text
def resource_to_con_text(resource: Dict[str, Any]) -> str:
"""Backward-compatible public helper for the typed soft-constraint view."""
if not isinstance(resource.get("constraint"), dict):
return str(resource.get("resource_id") or resource.get("id") or "")
return build_constraint_profile(resource).soft_constraint_text
def embed_passages(texts: List[str], model: SentenceTransformer) -> np.ndarray:
vectors = model.encode(
prepare_passage_texts(texts),
batch_size=16,
show_progress_bar=True,
normalize_embeddings=True,
)
return np.asarray(vectors, dtype="float32")
def embed_texts(texts: List[str], model: SentenceTransformer) -> np.ndarray:
"""Backward-compatible alias with corrected passage semantics."""
return embed_passages(texts, model)
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json_sha256(payload: Any) -> str:
encoded = json.dumps(
payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _validate_vector_contract(
cap_vectors: np.ndarray,
con_vectors: np.ndarray,
*,
expected_rows: int,
expected_dimension: int,
) -> None:
if cap_vectors.shape != con_vectors.shape:
raise ValueError(
"Capability/constraint vector shape mismatch: "
f"{cap_vectors.shape} != {con_vectors.shape}"
)
if cap_vectors.shape != (expected_rows, expected_dimension):
raise ValueError(
"Embedding output/index shape mismatch: "
f"{cap_vectors.shape} != {(expected_rows, expected_dimension)}"
)
if not np.isfinite(cap_vectors).all() or not np.isfinite(con_vectors).all():
raise ValueError("Embedding vectors contain non-finite values")
for name, vectors in (("capability", cap_vectors), ("constraint", con_vectors)):
norms = np.linalg.norm(vectors, axis=1)
if not np.allclose(norms, np.ones_like(norms), rtol=1e-4, atol=1e-4):
raise ValueError(f"{name} vectors are not L2-normalized")
def _commit_index_generation(
*,
output_dir: Path,
cap_index: faiss.Index,
con_index: faiss.Index,
metadata: Dict[str, Any],
audit: Dict[str, Any],
source_seal: Dict[str, Any],
) -> Dict[str, Any]:
"""Commit a complete index generation with a last-written hash manifest."""
output_dir.mkdir(parents=True, exist_ok=True)
temporary_token = uuid.uuid4().hex
destinations = {
CAP_INDEX_FILE.name: output_dir / CAP_INDEX_FILE.name,
CON_INDEX_FILE.name: output_dir / CON_INDEX_FILE.name,
METADATA_FILE.name: output_dir / METADATA_FILE.name,
AUDIT_FILE.name: output_dir / AUDIT_FILE.name,
}
temporary = {
name: destination.with_name(f".{destination.name}.{temporary_token}.tmp")
for name, destination in destinations.items()
}
manifest_path = output_dir / BUILD_MANIFEST_FILE.name
manifest_tmp = manifest_path.with_name(
f".{manifest_path.name}.{temporary_token}.tmp"
)
try:
faiss.write_index(cap_index, str(temporary[CAP_INDEX_FILE.name]))
faiss.write_index(con_index, str(temporary[CON_INDEX_FILE.name]))
temporary[METADATA_FILE.name].write_bytes(
pickle.dumps(metadata, protocol=pickle.HIGHEST_PROTOCOL)
)
temporary[AUDIT_FILE.name].write_text(
json.dumps(audit, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
file_hashes = {
name: _sha256_file(path) for name, path in temporary.items()
}
generation_id = _canonical_json_sha256(
{
"embedding_configuration_sha256": metadata[
"embedding_runtime_config"
]["configuration_sha256"],
"embedding_identity_sha256": metadata[
"embedding_runtime_identity"
]["identity_sha256"],
"resource_order_sha256": metadata["resource_order_sha256"],
"release_source_seal_sha256": source_seal["seal_sha256"],
"files_sha256": file_hashes,
}
)[:32]
manifest = {
"protocol": "sgar-retrieval-index-build-v1",
"generation_id": generation_id,
"embedding_configuration_sha256": metadata[
"embedding_runtime_config"
]["configuration_sha256"],
"embedding_identity_sha256": metadata[
"embedding_runtime_identity"
]["identity_sha256"],
"embedding_runtime_identity_v2_sha256": metadata[
"embedding_runtime_identity_v2"
]["identity_sha256"],
"index_meta_v2_sha256": metadata["index_meta_v2"][
"index_meta_sha256"
],
"resource_count": len(metadata["idx_to_id"]),
"resource_order_sha256": metadata["resource_order_sha256"],
"dimension": int(metadata["dim"]),
"release_source_seal": source_seal_reference(source_seal),
"build_code_sha256": _sha256_file(Path(__file__).resolve()),
"files_sha256": file_hashes,
}
manifest["manifest_sha256"] = _canonical_json_sha256(manifest)
manifest_tmp.write_text(
json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
encoding="utf-8",
)
for name, destination in destinations.items():
temporary[name].replace(destination)
manifest_tmp.replace(manifest_path)
return manifest
finally:
for path in (*temporary.values(), manifest_tmp):
if path.exists():
path.unlink()
def build_faiss_index(vectors: np.ndarray) -> faiss.Index:
if len(vectors.shape) != 2 or vectors.shape[0] == 0:
raise ValueError("Cannot build a FAISS index from an empty/non-matrix vector set")
index = faiss.IndexFlatIP(int(vectors.shape[1]))
index.add(vectors)
return index
def build_metadata(
resources: List[Dict[str, Any]],
cap_vectors: np.ndarray,
con_vectors: np.ndarray,
duplicate_aliases: Dict[str, str],
*,
readiness: Dict[str, Any],
readiness_hash: str,
embedding_config: EmbeddingRuntimeConfigV1 | None = None,
embedding_identity: Dict[str, Any] | None = None,
embedding_runtime_identity_v2: Dict[str, Any] | None = None,
source_seal: Dict[str, Any] | None = None,
) -> Dict[str, Any]:
profiles = [build_constraint_profile(resource) for resource in resources]
runtime_config = embedding_config or load_embedding_release_config(
EMBEDDING_RELEASE_CONFIG_FILE
)
resource_ids = tuple(resource["resource_id"] for resource in resources)
runtime_identity_v2 = (
EmbeddingRuntimeIdentityV2.model_validate(embedding_runtime_identity_v2)
if embedding_runtime_identity_v2
else None
)
index_meta_v2 = (
IndexMetaV2(
embedding_runtime_identity=runtime_identity_v2,
resource_manifest_sha256=_canonical_json_sha256(resources),
resource_id_order=resource_ids,
resource_id_order_sha256=_canonical_json_sha256(resource_ids),
capability_vectors_sha256=hashlib.sha256(
np.ascontiguousarray(cap_vectors, dtype="float32").tobytes()
).hexdigest(),
constraint_vectors_sha256=hashlib.sha256(
np.ascontiguousarray(con_vectors, dtype="float32").tobytes()
).hexdigest(),
faiss_dimension=int(cap_vectors.shape[1]),
vector_count=int(cap_vectors.shape[0]),
).model_dump(mode="json")
if runtime_identity_v2 is not None
else {}
)
return {
"schema_version": 3,
"profile_version": PROFILE_VERSION,
"embedding_model": runtime_config.model_id,
"bge_prefix": (
runtime_config.capability_query_instruction
if runtime_config.family == "bge"
else ""
),
"query_prefix": runtime_config.capability_query_instruction,
"passage_prefix": runtime_config.document_instruction,
"encoding_policy_version": runtime_config.protocol,
"embedding_runtime_config": runtime_config.model_dump(mode="json"),
"embedding_runtime_identity": embedding_identity or {},
"embedding_runtime_identity_v2": embedding_runtime_identity_v2 or {},
"index_meta_v2": index_meta_v2,
"catalog_sha256": readiness.get("catalog_sha256"),
"readiness_report_sha256": readiness_hash,
"model_health_sha256": readiness.get("evidence", {}).get("model_health_sha256"),
"docker_image": readiness.get("evidence", {}).get("docker_image"),
"docker_image_id": readiness.get("evidence", {}).get("docker_image_id"),
"docker_repo_digests": readiness.get("evidence", {}).get("docker_repo_digests", []),
"id_to_resource": {resource["resource_id"]: resource for resource in resources},
"idx_to_id": {index: resource["resource_id"] for index, resource in enumerate(resources)},
"id_to_idx": {resource["resource_id"]: index for index, resource in enumerate(resources)},
"resource_order_sha256": _canonical_json_sha256(
resource_ids
),
"cap_vectors": cap_vectors.tolist(),
"con_vectors": con_vectors.tolist(),
"dim": int(cap_vectors.shape[1]),
"capability_texts": {
profile.resource_id: profile.capability_text for profile in profiles
},
"constraint_texts": {
profile.resource_id: profile.soft_constraint_text for profile in profiles
},
"hard_requirements": {
profile.resource_id: profile.hard_requirements for profile in profiles
},
"utility_profiles": {
profile.resource_id: profile.utility_profile for profile in profiles
},
"manifest_hashes": {
profile.resource_id: profile.manifest_hash for profile in profiles
},
"duplicate_resource_aliases": duplicate_aliases,
"release_source_seal": (
source_seal_reference(source_seal) if source_seal is not None else None
),
}
def main() -> None:
parser = argparse.ArgumentParser(description="Build typed S-GAR dual-vector indices")
parser.add_argument(
"--allow-core-profile-errors",
action="store_true",
help="Write an index despite Model/Agent/Skill/Resource validation errors.",
)
parser.add_argument(
"--release-source-seal",
type=Path,
required=True,
help="ReleaseSourceSealV1 that binds this complete fresh index generation.",
)
parser.add_argument(
"--embedding-release-config",
type=Path,
default=EMBEDDING_RELEASE_CONFIG_FILE,
help="Fixed Qwen release configuration; model weights must already be local.",
)
parser.add_argument(
"--resources-file",
type=Path,
default=RESOURCES_FILE,
help="Readiness-gated effective catalog to index.",
)
parser.add_argument(
"--readiness-report",
type=Path,
default=READINESS_FILE,
help="RC1 readiness report authorizing every indexed resource.",
)
parser.add_argument(
"--output-dir",
type=Path,
required=True,
help="New empty version directory under SGAR_RELEASE_STORAGE_ROOT; formal promotion is separate.",
)
args = parser.parse_args()
source_seal_path = args.release_source_seal.expanduser().resolve()
source_seal = load_and_verify_source_seal(
source_seal_path,
project_root=PROJECT_ROOT,
allowed_stages=("release",),
)
embedding_config = load_embedding_release_config(
args.embedding_release_config.resolve()
)
output_dir = require_release_storage_path(
args.output_dir, code="formal_index_staging_directory_outside_storage_root"
)
if output_dir == INDEX_DIR.resolve():
raise ValueError("formal_index_build_must_use_versioned_staging_directory")
if output_dir.exists() and any(output_dir.iterdir()):
raise ValueError("versioned_index_output_directory_not_empty")
cap_index_file = output_dir / CAP_INDEX_FILE.name
con_index_file = output_dir / CON_INDEX_FILE.name
metadata_file = output_dir / METADATA_FILE.name
audit_file = output_dir / AUDIT_FILE.name
readiness_bytes = args.readiness_report.read_bytes()
readiness = json.loads(readiness_bytes.decode("utf-8-sig"))
allowed_ids = set(readiness.get("effective_resource_ids") or [])
from sgar_mvp.src.model_selection import require_admitted_candidate_models
model_health = json.loads((PROJECT_ROOT / "sgar_mvp/runtime_state/model_ready_state.json").read_text(encoding="utf-8-sig"))
if (readiness.get("evidence") or {}).get("model_health_sha256") != model_health.get("health_sha256"):
raise ValueError("index_readiness_health_identity_mismatch")
supplied_resources = load_resources(args.resources_file)
require_admitted_candidate_models(supplied_resources, model_health, root=PROJECT_ROOT)
raw_resources = filter_indexable_resources(supplied_resources)
canonical_resources = load_resources(CATALOG_FILE)
catalog_consistency = build_capability_consistency_report(
{
str(item["resource_id"]): item
for item in canonical_resources
}
)
if catalog_consistency.sealable_resource_count != catalog_consistency.resource_count:
failures = [
f"{item.resource_id}:{','.join(item.issue_codes)}"
for item in catalog_consistency.items
if not item.sealable
]
raise ValueError(
"Canonical resource catalog is not execution-ready and English: "
+ "; ".join(failures[:20])
)
actual_ids = {item["resource_id"] for item in raw_resources}
if actual_ids != allowed_ids:
missing = sorted(allowed_ids - actual_ids)
unexpected = sorted(actual_ids - allowed_ids)
raise ValueError(
"Effective catalog/readiness mismatch: "
f"missing={missing[:20]}, unexpected={unexpected[:20]}"
)
if any(
item.get("resource_type") == "Resource"
or item.get("type", {}).get("resource_type") == "Resource"
for item in raw_resources
):
raise ValueError("RC1 effective catalog must not contain Resource entries")
audit = validate_resource_profiles(raw_resources, PROJECT_ROOT)
resources, duplicate_aliases = deduplicate_exact_skill_packages(raw_resources)
audit["indexed_resource_count"] = len(resources)
audit["duplicate_resource_aliases"] = duplicate_aliases
if audit["errors"] and not args.allow_core_profile_errors:
examples = ", ".join(
f"{item['resource_id']}:{item['code']}" for item in audit["errors"][:10]
)
raise ValueError(
"Core retrieval-profile validation failed. "
f"No index generation was committed. Examples: {examples}"
)
profiles = [build_constraint_profile(resource) for resource in resources]
cap_texts = [profile.capability_text for profile in profiles]
con_texts = [profile.soft_constraint_text for profile in profiles]
print(f"[build_index] Profile version: {PROFILE_VERSION}")
print(f"[build_index] Exact duplicate Skill aliases: {len(duplicate_aliases)}")
print("[build_index] Resolving pinned local Qwen embedding model...")
embedding_runtime = LocalEmbeddingEncoder(embedding_config)
embedding_identity = embedding_runtime.identity()
embedding_identity_v2 = embedding_runtime.runtime_identity_v2()
print(
"[build_index] Loading offline embedding snapshot "
f"{embedding_identity.local_snapshot_locator}"
)
embedding_runtime.load()
print("[build_index] Encoding capability vectors...")
cap_vectors = embedding_runtime.encode_documents(cap_texts, batch_size=16)
print("[build_index] Encoding soft-constraint vectors...")
con_vectors = embedding_runtime.encode_documents(con_texts, batch_size=16)
_validate_vector_contract(
cap_vectors,
con_vectors,
expected_rows=len(resources),
expected_dimension=embedding_config.output_dimension,
)
cap_index = build_faiss_index(cap_vectors)
con_index = build_faiss_index(con_vectors)
metadata = build_metadata(
resources,
cap_vectors,
con_vectors,
duplicate_aliases,
readiness=readiness,
readiness_hash="sha256:" + hashlib.sha256(readiness_bytes).hexdigest(),
embedding_config=embedding_config,
embedding_identity=embedding_identity.model_dump(mode="json"),
embedding_runtime_identity_v2=embedding_identity_v2.model_dump(mode="json"),
source_seal=source_seal,
)
source_seal = load_and_verify_source_seal(
source_seal_path,
project_root=PROJECT_ROOT,
allowed_stages=("release",),
)
current_health = json.loads((PROJECT_ROOT / "sgar_mvp/runtime_state/model_ready_state.json").read_text(encoding="utf-8-sig"))
if current_health.get("health_sha256") != model_health["health_sha256"]:
raise ValueError("index_health_changed_during_build")
require_admitted_candidate_models(resources, current_health, root=PROJECT_ROOT)
manifest = _commit_index_generation(
output_dir=output_dir,
cap_index=cap_index,
con_index=con_index,
metadata=metadata,
audit=audit,
source_seal=source_seal,
)
print(
"[build_index] Saved "
f"{cap_index.ntotal} resources, dim={metadata['dim']}, "
f"profile={PROFILE_VERSION}"
)
print(f"[build_index] Audit: {audit_file}")
print(
"[build_index] Generation manifest: "
f"{output_dir / BUILD_MANIFEST_FILE.name} "
f"({manifest['manifest_sha256']})"
)
print(f"[build_index] Output directory: {output_dir}")
if __name__ == "__main__":
main()