ChatDev/execorchestra/scripts/embedding_server.py
2026-11-05 16:22:12 +08:00

115 lines
4.9 KiB
Python

"""Local, authenticated embedding endpoint. Model loading occurs only at startup."""
from contextlib import asynccontextmanager
import hmac
import os
from pathlib import Path
import threading
from fastapi import FastAPI, Header, HTTPException
from pydantic import BaseModel, ConfigDict, Field
MODEL_ID = 'sgar-embedding-4b'
REVISION = '5cf2132abc99cad020ac570b19d031efec650f2b'
DIMENSION = 2560
MAX_TOKENS = 32768
ROOT = Path(__file__).resolve().parents[1]
MODEL_PATH = Path(os.environ.get('SGAR_EMBEDDING_MODEL_PATH', str(ROOT / '.cache/embedding-model')))
state = {}
lock = threading.Lock()
@asynccontextmanager
async def lifespan(app):
if not os.environ.get('SLURM_JOB_ID') and os.environ.get('SGAR_EMBEDDING_DEDICATED_GPU') != '1':
raise RuntimeError('Use an active Slurm GPU allocation or explicitly set SGAR_EMBEDDING_DEDICATED_GPU=1 on a dedicated GPU server')
if not os.environ.get('SGAR_EMBEDDING_API_KEY'):
raise RuntimeError('SGAR_EMBEDDING_API_KEY is required')
import torch
from transformers import AutoModel, AutoTokenizer
if not torch.cuda.is_available():
raise RuntimeError('No allocated CUDA GPU is visible')
if not torch.cuda.is_bf16_supported():
raise RuntimeError('The allocated GPU does not support BF16')
tokenizer = AutoTokenizer.from_pretrained(
str(MODEL_PATH), local_files_only=True, trust_remote_code=False,
padding_side='left')
model = AutoModel.from_pretrained(
str(MODEL_PATH), local_files_only=True, trust_remote_code=False,
dtype=torch.bfloat16, attn_implementation='sdpa').to('cuda').eval()
if model.config.hidden_size != DIMENSION:
raise RuntimeError('Unexpected embedding dimension')
state.update(model=model, tokenizer=tokenizer)
try:
yield
finally:
state.clear()
del model
torch.cuda.empty_cache()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None)
class EmbeddingRequest(BaseModel):
model_config = ConfigDict(extra='forbid')
model: str
input: str | list[str]
encoding_format: str = 'float'
dimensions: int = Field(default=DIMENSION)
@app.get('/healthz')
def health():
if 'model' not in state:
raise HTTPException(503, 'Model is not ready')
return {'ready': True, 'model': MODEL_ID, 'revision': REVISION, 'dimensions': DIMENSION}
@app.post('/v1/embeddings')
def embeddings(body: EmbeddingRequest, authorization: str = Header(default='')):
key = os.environ.get('SGAR_EMBEDDING_API_KEY', '')
if not key or not hmac.compare_digest(authorization, 'Bearer ' + key):
raise HTTPException(401, 'Invalid API key')
if body.model != MODEL_ID:
raise HTTPException(400, 'Unsupported model')
if body.dimensions != DIMENSION or body.encoding_format != 'float':
raise HTTPException(400, 'Only 2560-dimensional float embeddings are supported')
inputs = [body.input] if isinstance(body.input, str) else body.input
if not inputs or len(inputs) > 64:
raise HTTPException(400, 'Provide between 1 and 64 input strings')
if sum(len(s) for s in inputs) > 8 * 1024 * 1024:
raise HTTPException(413, 'Input is too large')
import torch
import torch.nn.functional as functional
with lock, torch.inference_mode():
tokenizer, model = state['tokenizer'], state['model']
tokens = tokenizer(inputs, padding=False, truncation=False)
counts = [len(ids) for ids in tokens['input_ids']]
if any(n == 0 or n > MAX_TOKENS for n in counts):
raise HTTPException(400, 'Each input must contain 1 to 32768 tokens; no silent truncation')
results = []
# Fixed microbatch of one keeps 32K requests within a bounded memory footprint.
# The caller already adds query instructions; do not add a second prefix here.
try:
for i in range(len(inputs)):
batch = {k: torch.tensor([v[i]], device='cuda') for k, v in tokens.items()}
hidden = model(**batch).last_hidden_state
vector = functional.normalize(hidden[:, -1, :].float(), p=2, dim=1)
if not torch.isfinite(vector).all():
raise HTTPException(500, 'Non-finite embedding')
results.append({'object': 'embedding', 'index': i,
'embedding': vector[0].cpu().tolist()})
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
raise HTTPException(503, 'GPU memory exhausted; reduce input length') from None
total = sum(counts)
return {'object': 'list', 'data': results, 'model': MODEL_ID,
'usage': {'prompt_tokens': total, 'total_tokens': total}}
if __name__ == '__main__':
if not os.environ.get('SLURM_JOB_ID') and os.environ.get('SGAR_EMBEDDING_DEDICATED_GPU') != '1':
raise SystemExit('Use a Slurm allocation or explicitly select a dedicated GPU server')
import uvicorn
uvicorn.run(app, host='127.0.0.1', port=8001, workers=1, access_log=False)