import asyncio import os import time from concurrent.futures import ThreadPoolExecutor from textwrap import dedent from typing import List, Tuple, Union from uuid import uuid4 import torch from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from FlagEmbedding import BGEM3FlagModel from pydantic import BaseModel from starlette.responses import RedirectResponse from starlette.status import HTTP_504_GATEWAY_TIMEOUT _ = """ Path("/tmp/cache").mkdir(exist_ok=True) os.environ["HF_HOME"] = "/tmp/cache" os.environ["TRANSFORMERS_CACHE"] = "/tmp/cache" # does not quite work # """ batch_size = 2 # gpu batch_size in order of your available vram max_request = 10 # max request for future improvements on api calls / gpu batches (for now is pretty basic) max_length = 5000 # max context length for embeddings and passages in re-ranker max_q_length = 256 # max context lenght for questions in re-ranker request_flush_timeout = 0.1 # flush time out for future improvements on api calls / gpu batches (for now is pretty basic) rerank_weights = [0.4, 0.2, 0.4] # re-rank score weights request_time_out = 30 # Timeout threshold request_time_out = 1200 # Timeout threshold gpu_time_out = 5 # gpu processing timeout threshold gpu_time_out = 600 # gpu processing timeout threshold port = 3000 port = 7860 DEVICE = "cuda" if torch.cuda.is_available() else "cpu" os.environ["TZ"] = "Asia/Shanghai" try: time.tzset() # type: ignore # pylint: disable=no-member except Exception: # Windows print("Windows, cant run time.tzset()") class m3Wrapper: def __init__(self, model_name: str, device: str = DEVICE): """Init.""" self.model = BGEM3FlagModel( model_name, device=device, use_fp16=True if device != "cpu" else False ) def embed(self, sentences: List[str]) -> List[List[float]]: embeddings = self.model.encode( sentences, batch_size=batch_size, max_length=max_length )["dense_vecs"] embeddings = embeddings.tolist() return embeddings def rerank(self, sentence_pairs: List[Tuple[str, str]]) -> List[float]: scores = self.model.compute_score( sentence_pairs, batch_size=batch_size, max_query_length=max_q_length, max_passage_length=max_length, weights_for_different_modes=rerank_weights, )["colbert+sparse+dense"] return scores class EmbedRequest(BaseModel): sentences: List[str] class RerankRequest(BaseModel): sentence_pairs: List[Tuple[str, str]] class EmbedResponse(BaseModel): embeddings: List[List[float]] class RerankResponse(BaseModel): scores: List[float] class RequestProcessor: def __init__( self, model: m3Wrapper, max_request_to_flush: int, accumulation_timeout: float ): """Init.""" self.model = model self.max_batch_size = max_request_to_flush self.accumulation_timeout = accumulation_timeout self.queue = asyncio.Queue() self.response_futures = {} self.processing_loop_task = None self.processing_loop_started = False # Processing pool flag lazy init state self.executor = ThreadPoolExecutor() # Thread pool self.gpu_lock = asyncio.Semaphore(1) # Sem for gpu sync usage async def ensure_processing_loop_started(self): if not self.processing_loop_started: print("starting processing_loop") self.processing_loop_task = asyncio.create_task(self.processing_loop()) self.processing_loop_started = True async def processing_loop(self): while True: requests, request_types, request_ids = [], [], [] start_time = asyncio.get_event_loop().time() while len(requests) < self.max_batch_size: timeout = self.accumulation_timeout - ( asyncio.get_event_loop().time() - start_time ) if timeout <= 0: break try: req_data, req_type, req_id = await asyncio.wait_for( self.queue.get(), timeout=timeout ) requests.append(req_data) request_types.append(req_type) request_ids.append(req_id) except asyncio.TimeoutError: break if requests: await self.process_requests_by_type( requests, request_types, request_ids ) async def process_requests_by_type(self, requests, request_types, request_ids): tasks = [] for request_data, request_type, request_id in zip( requests, request_types, request_ids ): if request_type == "embed": task = asyncio.create_task( self.run_with_semaphore( self.model.embed, request_data.sentences, request_id ) ) else: # 'rerank' task = asyncio.create_task( self.run_with_semaphore( self.model.rerank, request_data.sentence_pairs, request_id ) ) tasks.append(task) await asyncio.gather(*tasks) async def run_with_semaphore(self, func, data, request_id): async with self.gpu_lock: # Wait for sem future = self.executor.submit(func, data) try: result = await asyncio.wait_for( asyncio.wrap_future(future), timeout=gpu_time_out ) self.response_futures[request_id].set_result(result) except asyncio.TimeoutError: self.response_futures[request_id].set_exception( TimeoutError("GPU processing timeout") ) except Exception as e: self.response_futures[request_id].set_exception(e) async def process_request( self, request_data: Union[EmbedRequest, RerankRequest], request_type: str ): try: await self.ensure_processing_loop_started() request_id = str(uuid4()) self.response_futures[request_id] = asyncio.Future() await self.queue.put((request_data, request_type, request_id)) return await self.response_futures[request_id] except Exception as e: raise HTTPException(status_code=500, detail=f"Internal Server Error: {e}") description = dedent( r""" embed example: ```bash curl -X 'POST' \ 'https://evgensoft-baai-m3.hf.space/embed/' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ "sentences": [ "string", "string1" ] }' ``` rerank example: ```bash ... ``` """ ) app = FastAPI( title="Serving deepvk/USER-bge-m3 embed and rerank", # description="Swagger UI at https://evgensoft-baai-m3.hf.space/docs", description=description, version="0.1.0a0", ) # Initialize the model and request processor model = m3Wrapper("deepvk/USER-bge-m3") processor = RequestProcessor( model, accumulation_timeout=request_flush_timeout, max_request_to_flush=max_request ) # Adding a middleware returning a 504 error if the request processing time is above a certain threshold @app.middleware("http") async def timeout_middleware(request: Request, call_next): try: start_time = time.time() return await asyncio.wait_for(call_next(request), timeout=request_time_out) except asyncio.TimeoutError: process_time = time.time() - start_time return JSONResponse( { "detail": "Request processing time excedeed limit", "processing_time": process_time, }, status_code=HTTP_504_GATEWAY_TIMEOUT, ) @app.get("/") async def landing(): """Define landing page.""" # return "Swagger UI at https://evgensoft-baai-m3.hf.space/docs" return RedirectResponse("/docs") @app.post("/embed/", response_model=EmbedResponse) async def get_embeddings(request: EmbedRequest): embeddings = await processor.process_request(request, "embed") return EmbedResponse(embeddings=embeddings) @app.post("/rerank/", response_model=RerankResponse) async def rerank(request: RerankRequest): scores = await processor.process_request(request, "rerank") return RerankResponse(scores=scores) if __name__ == "__main__": import uvicorn print("started") uvicorn.run(app, host="0.0.0.0", port=port)