mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
40 lines
1.4 KiB
Python
40 lines
1.4 KiB
Python
"""Download the pinned embedding assets without starting a service."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def download(*, tokenizer_only=False):
|
|
from huggingface_hub import snapshot_download
|
|
from transformers import AutoTokenizer
|
|
|
|
runtime = json.loads((ROOT / "configs/embedding_release.example.json").read_text(
|
|
encoding="utf-8"))["runtime"]
|
|
model, revision = runtime["model_id"], runtime["revision"]
|
|
tokenizer_path = ROOT / ".cache/embedding-tokenizer"
|
|
if tokenizer_only:
|
|
tokenizer = AutoTokenizer.from_pretrained(model, revision=revision)
|
|
else:
|
|
model_path = ROOT / ".cache/embedding-model"
|
|
snapshot_download(model, revision=revision, local_dir=str(model_path))
|
|
tokenizer = AutoTokenizer.from_pretrained(str(model_path), local_files_only=True)
|
|
tokenizer.save_pretrained(str(tokenizer_path))
|
|
print(f"Embedding assets ready: {model} @ {revision}")
|
|
print(f"Tokenizer: {tokenizer_path}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--tokenizer-only", action="store_true",
|
|
help="Use an existing embedding endpoint; download only its tokenizer.")
|
|
args = parser.parse_args()
|
|
download(tokenizer_only=args.tokenizer_only)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|