apis/chat_api.py

import argparse
import os
import sys
import time
import uvicorn
import requests
import asyncio
import logging

from pathlib import Path
from fastapi import FastAPI, Depends, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from pydantic import BaseModel, Field
from typing import Union, List, Dict, Any
from sse_starlette.sse import EventSourceResponse, ServerSentEvent
from utils.logger import logger
from networks.message_streamer import MessageStreamer
from messagers.message_composer import MessageComposer
from mocks.stream_chat_mocker import stream_chat_mock

from fastapi.middleware.cors import CORSMiddleware


class EmbeddingResponseItem(BaseModel):
    object: str = "embedding"
    index: int
    embedding: List[List[float]] 

class EmbeddingResponse(BaseModel):
    object: str = "list"
    data: List[EmbeddingResponseItem]
    model: str
    usage: Dict[str, Any]


class ChatAPIApp:
    def __init__(self):
        self.app = FastAPI(
            docs_url="/",
            title="HuggingFace LLM API",
            swagger_ui_parameters={"defaultModelsExpandDepth": -1},
            version="1.0",
        )
        self.setup_routes()

        self.app.add_middleware(
            CORSMiddleware,
            allow_origins=["*"],  # You can specify specific origins here
            allow_credentials=True,
            allow_methods=["*"],  # Or specify just the methods you need: ["GET", "POST"]
            allow_headers=["*"],  # Or specify headers you need
        )

    def get_available_models(self):
        # https://platform.openai.com/docs/api-reference/models/list
        # ANCHOR[id=available-models]: Available models
        current_time = int(time.time())
        self.available_models = {
            "object": "list",
            "data": [
                {
                    "id": "mixtral-8x7b",
                    "description": "[mistralai/Mixtral-8x7B-Instruct-v0.1]: https://huggingface.co./mistralai/Mixtral-8x7B-Instruct-v0.1",
                    "object": "model",
                    "created": current_time,
                    "owned_by": "mistralai",
                },
                {
                    "id": "mistral-7b",
                    "description": "[mistralai/Mistral-7B-Instruct-v0.2]: https://huggingface.co./mistralai/Mistral-7B-Instruct-v0.2",
                    "object": "model",
                    "created": current_time,
                    "owned_by": "mistralai",
                },
                {
                    "id": "nous-mixtral-8x7b",
                    "description": "[NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO]: https://huggingface.co./NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
                    "object": "model",
                    "created": current_time,
                    "owned_by": "NousResearch",
                },
                {
                    "id": "gemma-7b",
                    "description": "[google/gemma-7b-it]: https://huggingface.co./google/gemma-7b-it",
                    "object": "model",
                    "created": current_time,
                    "owned_by": "Google",
                },
                {
                    "id": "codellama-7b",
                    "description": "[codellama/CodeLlama-7b-hf]: https://huggingface.co./codellama/CodeLlama-7b-hf",
                    "object": "model",
                    "created": current_time,
                    "owned_by": "codellama",
                },
                {
                    "id": "bert-base-uncased",
                    "description": "[google-bert/bert-base-uncased]: https://huggingface.co./google-bert/bert-base-uncased",
                    "object": "embedding",
                    "created": current_time,
                    "owned_by": "google",
                },
            ],
        }
        return self.available_models

    def extract_api_key(
        credentials: HTTPAuthorizationCredentials = Depends(
            HTTPBearer(auto_error=False)
        ),
    ):
        api_key = None
        if credentials:
            api_key = credentials.credentials
        else:
            api_key = os.getenv("HF_TOKEN")

        if api_key:
            if api_key.startswith("hf_"):
                return api_key
            else:
                logger.warn(f"Invalid HF Token!")
        else:
            logger.warn("Not provide HF Token!")
        return None

    class QueryRequest(BaseModel):
        input: str
        model: str = Field(default="bert-base-uncased")
        encoding_format: str

    class ChatCompletionsPostItem(BaseModel):
        model: str = Field(
            default="mixtral-8x7b",
            description="(str) `mixtral-8x7b`",
        )
        messages: list = Field(
            default=[{"role": "user", "content": "Hello, who are you?"}],
            description="(list) Messages",
        )
        temperature: Union[float, None] = Field(
            default=0.5,
            description="(float) Temperature",
        )
        top_p: Union[float, None] = Field(
            default=0.95,
            description="(float) top p",
        )
        max_tokens: Union[int, None] = Field(
            default=-1,
            description="(int) Max tokens",
        )
        use_cache: bool = Field(
            default=False,
            description="(bool) Use cache",
        )
        stream: bool = Field(
            default=False,
            description="(bool) Stream",
        )

    def chat_completions(
        self, item: ChatCompletionsPostItem, api_key: str = Depends(extract_api_key)
    ):
        streamer = MessageStreamer(model=item.model)
        composer = MessageComposer(model=item.model)
        composer.merge(messages=item.messages)
        # streamer.chat = stream_chat_mock

        stream_response = streamer.chat_response(
            prompt=composer.merged_str,
            temperature=item.temperature,
            top_p=item.top_p,
            max_new_tokens=item.max_tokens,
            api_key=api_key,
            use_cache=item.use_cache,
        )
        if item.stream:
            event_source_response = EventSourceResponse(
                streamer.chat_return_generator(stream_response),
                media_type="text/event-stream",
                ping=2000,
                ping_message_factory=lambda: ServerSentEvent(**{"comment": ""}),
            )
            return event_source_response
        else:
            data_response = streamer.chat_return_dict(stream_response)
            return data_response

    async def embedding(self, request: QueryRequest, api_key: str = Depends(extract_api_key)):
        api_url = f"https://api-inference.huggingface.co/pipeline/feature-extraction/{request.model}"
        headers = {"Authorization": f"Bearer {api_key}"}
        
        response = await requests.post(api_url, headers=headers, json={"inputs": request.input})
        result = response.json()
        
        if "error" in result:
            logging.error(f"Error from Hugging Face API: {result['error']}")
            error_detail = result.get('error', 'No detailed error message provided.')
            raise HTTPException(status_code=503, detail=f"The model is currently loading, please re-run the query. Detail: {error_detail}")
        
        if isinstance(result, list) and len(result) > 0 and isinstance(result[0], list):
            flattened_embeddings = [item for sublist in result for item in sublist]  # Flatten list of lists
            data = [{"object": "embedding", "index": i, "embedding": embedding} for i, embedding in enumerate(flattened_embeddings)]
            return EmbeddingResponse(
                object="list",
                data=data,
                model=request.model,
                usage={"prompt_tokens": len(request.input), "total_tokens": len(request.input)}
            )
        else:
            logging.error(f"Unexpected response format: {result}")
            raise HTTPException(status_code=500, detail="Unexpected response format.")

    def setup_routes(self):
        for prefix in ["", "/v1", "/api", "/api/v1"]:
            if prefix in ["/api/v1"]:
                include_in_schema = True
            else:
                include_in_schema = False

            self.app.get(
                prefix + "/models",
                summary="Get available models",
                include_in_schema=include_in_schema,
            )(self.get_available_models)

            self.app.post(
                prefix + "/chat/completions",
                summary="Chat completions in conversation session",
                include_in_schema=include_in_schema,
            )(self.chat_completions)

            self.app.post(
                prefix + "/embeddings",  # Use the specific prefix for this route
                summary="Generate embeddings for the given texts",
                include_in_schema=include_in_schema,
                response_model=EmbeddingResponse  # Adapt based on your actual response model
            )(self.embedding)


class ArgParser(argparse.ArgumentParser):
    def __init__(self, *args, **kwargs):
        super(ArgParser, self).__init__(*args, **kwargs)

        self.add_argument(
            "-s",
            "--server",
            type=str,
            default="0.0.0.0",
            help="Server IP for HF LLM Chat API",
        )
        self.add_argument(
            "-p",
            "--port",
            type=int,
            default=23333,
            help="Server Port for HF LLM Chat API",
        )

        self.add_argument(
            "-d",
            "--dev",
            default=False,
            action="store_true",
            help="Run in dev mode",
        )

        self.args = self.parse_args(sys.argv[1:])


app = ChatAPIApp().app

if __name__ == "__main__":
    args = ArgParser().args
    if args.dev:
        uvicorn.run("__main__:app", host=args.server, port=args.port, reload=True)
    else:
        uvicorn.run("__main__:app", host=args.server, port=args.port, reload=False)

    # python -m apis.chat_api      # [Docker] on product mode
    # python -m apis.chat_api -d   # [Dev]    on develop mode