ChatDev/execorchestra/scripts/download_embedding.py
2026-11-05 16:22:12 +08:00

40 lines
1.4 KiB
Python

"""Download the pinned embedding assets without starting a service."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def download(*, tokenizer_only=False):
from huggingface_hub import snapshot_download
from transformers import AutoTokenizer
runtime = json.loads((ROOT / "configs/embedding_release.example.json").read_text(
encoding="utf-8"))["runtime"]
model, revision = runtime["model_id"], runtime["revision"]
tokenizer_path = ROOT / ".cache/embedding-tokenizer"
if tokenizer_only:
tokenizer = AutoTokenizer.from_pretrained(model, revision=revision)
else:
model_path = ROOT / ".cache/embedding-model"
snapshot_download(model, revision=revision, local_dir=str(model_path))
tokenizer = AutoTokenizer.from_pretrained(str(model_path), local_files_only=True)
tokenizer.save_pretrained(str(tokenizer_path))
print(f"Embedding assets ready: {model} @ {revision}")
print(f"Tokenizer: {tokenizer_path}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--tokenizer-only", action="store_true",
help="Use an existing embedding endpoint; download only its tokenizer.")
args = parser.parse_args()
download(tokenizer_only=args.tokenizer_only)
if __name__ == "__main__":
main()