import gradio as gr from openai import OpenAI import os # Retrieve the access token from the environment variable ACCESS_TOKEN = os.getenv("HF_TOKEN") print("Access token loaded.") # Initialize the OpenAI client with the Hugging Face Inference API endpoint client = OpenAI( base_url="https://api-inference.huggingface.co/v1/", api_key=ACCESS_TOKEN, ) print("OpenAI client initialized.") # We'll define a list of placeholder featured models for demonstration. # In real usage, replace them with actual model names available on Hugging Face. models_list = [ "PlaceholderModel1", "PlaceholderModel2", "PlaceholderModel3", "PlaceholderModel4", "PlaceholderModel5" ] def filter_featured_models(search_term): """ Filters the 'models_list' based on text entered in the search box. Returns a gr.update object that changes the choices available in the 'featured_models_radio'. """ filtered = [m for m in models_list if search_term.lower() in m.lower()] return gr.update(choices=filtered) def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, frequency_penalty, seed, custom_model, selected_model ): """ This function handles the chatbot response. It takes in: - message: the user's new message - history: the list of previous messages, each as a tuple (user_msg, assistant_msg) - system_message: the system prompt - max_tokens: the maximum number of tokens to generate in the response - temperature: sampling temperature - top_p: top-p (nucleus) sampling - frequency_penalty: penalize repeated tokens in the output - seed: a fixed seed for reproducibility; -1 will mean 'random' - custom_model: a custom Hugging Face model name (if any) - selected_model: a model name chosen from the featured models radio button """ print(f"Received message: {message}") print(f"History: {history}") print(f"System message: {system_message}") print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}") print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}") print(f"Custom model: {custom_model}") print(f"Selected featured model: {selected_model}") # Convert seed to None if -1 (meaning random) if seed == -1: seed = None # Construct the messages array required by the API messages = [{"role": "system", "content": system_message}] # Add conversation history to the context for val in history: user_part = val[0] assistant_part = val[1] if user_part: messages.append({"role": "user", "content": user_part}) print(f"Added user message to context: {user_part}") if assistant_part: messages.append({"role": "assistant", "content": assistant_part}) print(f"Added assistant message to context: {assistant_part}") # Append the latest user message messages.append({"role": "user", "content": message}) # Decide which model to use: # 1) If the user provided a custom model, use it. # 2) Else if they chose a featured model, use it. # 3) Otherwise, fall back to a default model. if custom_model.strip() != "": model_to_use = custom_model.strip() elif selected_model is not None and selected_model.strip() != "": model_to_use = selected_model.strip() else: model_to_use = "meta-llama/Llama-3.3-70B-Instruct" # Default fallback print(f"Model selected for inference: {model_to_use}") # Start with an empty string to build the response as tokens stream in response = "" print("Sending request to OpenAI API.") # Make the streaming request to the HF Inference API via openai-like client for message_chunk in client.chat.completions.create( model=model_to_use, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p, frequency_penalty=frequency_penalty, seed=seed, messages=messages, ): # Extract the token text from the response chunk token_text = message_chunk.choices[0].delta.content print(f"Received token: {token_text}") response += token_text # Yield the partial response to Gradio so it can display in real-time yield response print("Completed response generation.") ######################## # GRADIO APP LAYOUT ######################## # We’ll build a custom Blocks layout so we can have: # - A Featured Models accordion with a search box # - Our ChatInterface to handle the conversation # - Additional sliders and textboxes for settings (like the original code) ######################## with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo: gr.Markdown("## Serverless Text Generation Hub") gr.Markdown( "An all-in-one UI for chatting with text-generation models on Hugging Face's Inference API." ) # We keep a Chatbot component for the conversation display chatbot = gr.Chatbot(height=600, label="Chat Preview") # Textbox for system message system_message_box = gr.Textbox( value="", label="System Message", placeholder="Enter a system prompt if you want (optional).", ) # Slider for max_tokens max_tokens_slider = gr.Slider( minimum=1, maximum=4096, value=512, step=1, label="Max new tokens", ) # Slider for temperature temperature_slider = gr.Slider( minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature", ) # Slider for top_p top_p_slider = gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-P", ) # Slider for frequency penalty freq_penalty_slider = gr.Slider( minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty", ) # Slider for seed seed_slider = gr.Slider( minimum=-1, maximum=65535, # Arbitrary upper limit for demonstration value=-1, step=1, label="Seed (-1 for random)", ) # Custom Model textbox custom_model_box = gr.Textbox( value="", label="Custom Model", info="(Optional) Provide a custom Hugging Face model path. This will override the selected Featured Model if not empty." ) # Accordion for featured models with gr.Accordion("Featured Models", open=False): # Textbox for filtering the featured models model_search_box = gr.Textbox( label="Filter Models", placeholder="Search for a featured model...", lines=1, ) # Radio for selecting the desired model featured_models_radio = gr.Radio( label="Select a featured model below", choices=models_list, # Start with the entire list value=None, # No default interactive=True ) # We connect the model_search_box to the filter function model_search_box.change( filter_featured_models, inputs=model_search_box, outputs=featured_models_radio ) # Now we create our ChatInterface # We pass all the extra components as additional_inputs interface = gr.ChatInterface( fn=respond, chatbot=chatbot, additional_inputs=[ system_message_box, max_tokens_slider, temperature_slider, top_p_slider, freq_penalty_slider, seed_slider, custom_model_box, featured_models_radio ], theme="Nymbo/Nymbo_Theme", title="Serverless TextGen Hub with Featured Models", description=( "Use the sliders and textboxes to control generation parameters. " "Pick a model from 'Featured Models' or specify a custom model path." ), # Fill the screen height fill_height=True ) # If you want the script to be directly executable, launch the demo here: if __name__ == "__main__": print("Launching the demo application...") demo.launch()