porting-app-poc (#732)

- ported new app.py [wip] (a03f0fa73833144d05ac2cd45478474c88943b22)
- debugging the codebase (1489ff16db959ab5e96c6a0d454d92151f4772b2)
- added license search (d04186534937ac42efcddc9476b41dcf95aa5e6f)

app.py

@@ -1,10 +1,11 @@
 import os
-import time
+import pandas as pd
 import logging
+import time
 import gradio as gr
-import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from gradio_space_ci import enable_space_ci
 
 from src.display.about import (
@@ -49,14 +50,12 @@ from src.submission.submit import add_new_eval
 from src.tools.collections import update_collections
 from src.tools.plots import create_metric_plot_obj, create_plot_df, create_scores_df
 
-
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Start ephemeral Spaces on PRs (see config in README.md)
 enable_space_ci()
 
-
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
@@ -143,140 +142,6 @@ def load_and_create_plots():
     return plot_df
 
 
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    hide_models: list,
-    query: str,
-):
-    filtered_df = filter_models(
-        df=hidden_df,
-        type_query=type_query,
-        size_query=size_query,
-        precision_query=precision_query,
-        hide_models=hide_models,
-    )
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
-    return df
-
-
-def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
-    query = request.query_params.get("query") or ""
-    return (
-        query,
-        query,
-    )  # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
-
-
-def search_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.fullname.name].str.contains(query, case=False, na=False))]
-
-def search_license(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[df[AutoEvalColumn.license.name].str.contains(query, case=False, na=False)]
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-    dummy_col = [AutoEvalColumn.fullname.name]
-    filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col]
-    return filtered_df
-
-def filter_queries(query: str, df: pd.DataFrame):
-    tmp_result_df = []
-
-    # Empty query return the same df
-    if query == "":
-        return df
-
-    # all_queries = [q.strip() for q in query.split(";")]
-    # license_queries = []
-    all_queries = [q.strip() for q in query.split(";") if q.strip() != ""]
-    model_queries = [q for q in all_queries if not q.startswith("licence")]
-    license_queries_raw = [q for q in all_queries if q.startswith("license")]
-    license_queries = [
-        q.replace("license:", "").strip() for q in license_queries_raw if q.replace("license:", "").strip() != ""
-    ]
-
-    # Handling model name search
-    for query in model_queries:
-        tmp_df = search_model(df, query)
-        if len(tmp_df) > 0:
-            tmp_result_df.append(tmp_df)
-
-    if not tmp_result_df and not license_queries:
-        # Nothing is found, no license_queries -> return empty df
-        return pd.DataFrame(columns=df.columns)
-
-    if tmp_result_df:
-        df = pd.concat(tmp_result_df)
-        df = df.drop_duplicates(
-            subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-        )
-
-    if not license_queries:
-        return df
-
-    # Handling license search
-    tmp_result_df = []
-    for query in license_queries:
-        tmp_df = search_license(df, query)
-        if len(tmp_df) > 0:
-            tmp_result_df.append(tmp_df)
-
-    if not tmp_result_df:
-        # Nothing is found, return empty df
-        return pd.DataFrame(columns=df.columns)
-
-    df = pd.concat(tmp_result_df)
-    df = df.drop_duplicates(
-        subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-    )
-
-    return df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, hide_models: list
-) -> pd.DataFrame:
-    # Show all models
-    if "Private or deleted" in hide_models:
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-    else:
-        filtered_df = df
-
-    if "Contains a merge/moerge" in hide_models:
-        filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]
-
-    if "MoE" in hide_models:
-        filtered_df = filtered_df[filtered_df[AutoEvalColumn.moe.name] == False]
-
-    if "Flagged" in hide_models:
-        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
-leaderboard_df = filter_models(
-    df=leaderboard_df,
-    type_query=[t.to_str(" : ") for t in ModelType],
-    size_query=list(NUMERIC_INTERVALS.keys()),
-    precision_query=[i.value.name for i in Precision],
-    hide_models=["Private or deleted", "Contains a merge/moerge", "Flagged"],  # Deleted, merges, flagged, MoEs
-)
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -284,135 +149,40 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder="🔍 Search models or licenses (e.g., 'model_name; license: MIT') and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden and not c.dummy
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        hide_models = gr.CheckboxGroup(
-                            label="Hide models",
-                            choices=["Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
-                            value=["Private or deleted", "Contains a merge/moerge", "Flagged"],
-                            interactive=True,
-                        )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                    + [AutoEvalColumn.fullname.name]
+            leaderboard = Leaderboard(
+                value=leaderboard_df,
+                datatype=[c.type for c in fields(AutoEvalColumn)],
+                select_columns=SelectColumns(
+                    default_selection=[
+                        c.name
+                        for c in fields(AutoEvalColumn)
+                        if c.displayed_by_default
+                    ],
+                    cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
+                    label="Select Columns to Display:",
+                ),
+                search_columns=[
+                    AutoEvalColumn.model.name, 
+                    AutoEvalColumn.fullname.name, 
+                    AutoEvalColumn.license.name
                 ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    hide_models,
-                    search_bar,
+                hide_columns=[
+                    c.name
+                    for c in fields(AutoEvalColumn)
+                    if c.hidden
                 ],
-                leaderboard_table,
-            )
-
-            # Define a hidden component that will trigger a reload only if a query parameter has been set
-            hidden_search_bar = gr.Textbox(value="", visible=False)
-            hidden_search_bar.change(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    hide_models,
-                    search_bar,
+                filter_columns=[
+                    ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+                    ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+                    ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"),
+                    ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Private or deleted", default=True),
+                    ColumnFilter(AutoEvalColumn.merged.name, type="boolean", label="Contains a merge/moerge", default=True),
+                    ColumnFilter(AutoEvalColumn.moe.name, type="boolean", label="MoE", default=False),
+                    ColumnFilter(AutoEvalColumn.not_flagged.name, type="boolean", label="Flagged", default=True),
                 ],
-                leaderboard_table,
+                bool_checkboxgroup_label="Hide models"
             )
-            # Check query parameter once at startup and update search bar + hidden component
-            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
-
-            for selector in [
-                shown_columns,
-                filter_columns_type,
-                filter_columns_precision,
-                filter_columns_size,
-                hide_models,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        hide_models,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
+            
         with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):
             with gr.Row():
                 with gr.Column():
@@ -543,4 +313,4 @@ scheduler.add_job(restart_space, "interval", hours=3)  # restarted every 3h
 scheduler.add_job(update_dynamic_files, "interval", hours=2)  # launched every 2 hour
 scheduler.start()
 
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -44,9 +44,10 @@ tqdm = "4.65.0"
 transformers = "4.40.0"
 tokenizers = ">=0.15.0"
 gradio-space-ci = {git = "https://huggingface.co/spaces/Wauplin/gradio-space-ci", rev = "0.2.3"}
-gradio = "4.9.0"
+gradio = " 4.20.0"
 isort = "^5.13.2"
 ruff = "^0.3.5"
+gradio-leaderboard = "0.0.8"
 
 [build-system]
 requires = ["poetry-core"]

requirements.txt

@@ -13,4 +13,6 @@ sentencepiece
 tqdm==4.65.0
 transformers==4.40.0
 tokenizers>=0.15.0
-gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/[email protected] # CI !!!
+gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/[email protected] # CI !!!
+gradio==4.20.0
+gradio_leaderboard==0.0.8

src/display/utils.py

@@ -89,7 +89,7 @@ auto_eval_column_dict.append(
     ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)]
 )
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
+auto_eval_column_dict.append(["not_flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["fullname", ColumnContent, ColumnContent("fullname", "str", False, dummy=True)])
@@ -123,7 +123,7 @@ baseline_row = {
     AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.fullname.name: "baseline",
     AutoEvalColumn.model_type.name: "",
-    AutoEvalColumn.flagged.name: False,
+    AutoEvalColumn.not_flagged.name: False,
 }
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
@@ -148,7 +148,7 @@ human_baseline_row = {
     AutoEvalColumn.gsm8k.name: 100,
     AutoEvalColumn.fullname.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
-    AutoEvalColumn.flagged.name: False,
+    AutoEvalColumn.not_flagged.name: False,
 }
 
 

src/leaderboard/filter_models.py

@@ -133,11 +133,14 @@ DO_NOT_SUBMIT_MODELS = [
 def flag_models(leaderboard_data: list[dict]):
     """Flags models based on external criteria or flagged status."""
     for model_data in leaderboard_data:
-        # Merges and moes are flagged automatically
-        if model_data[AutoEvalColumn.flagged.name]:
-            flag_key = "merged"
-        else:
+        # If a model is not flagged, use its "fullname" as a key
+        if model_data[AutoEvalColumn.not_flagged.name]:
             flag_key = model_data[AutoEvalColumn.fullname.name]
+        else:
+             # Merges and moes are flagged 
+            flag_key = "merged"
+        
+        # Reverse the logic: Check for non-flagged models instead
         if flag_key in FLAGGED_MODELS:
             issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
             issue_link = model_hyperlink(
@@ -147,9 +150,9 @@ def flag_models(leaderboard_data: list[dict]):
             model_data[AutoEvalColumn.model.name] = (
                 f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
             )
-            model_data[AutoEvalColumn.flagged.name] = True
+            model_data[AutoEvalColumn.not_flagged.name] = False
         else:
-            model_data[AutoEvalColumn.flagged.name] = False
+            model_data[AutoEvalColumn.not_flagged.name] = True
 
 
 def remove_forbidden_models(leaderboard_data: list[dict]):

src/leaderboard/read_evals.py

@@ -37,7 +37,7 @@ class EvalResult:
     date: str = "" # submission date of request file
     still_on_hub: bool = True
     is_merge: bool = False
-    flagged: bool = False
+    not_flagged: bool = False
     status: str = "FINISHED"
     # List of tags, initialized to a new empty list for each instance to avoid the pitfalls of mutable default arguments.
     tags: List[str] = field(default_factory=list)
@@ -164,7 +164,7 @@ class EvalResult:
         self.tags = file_dict.get("tags", [])
         
         # Calculate `flagged` only if 'tags' is not empty and avoid calculating each time
-        self.flagged = "flagged" in self.tags
+        self.not_flagged = not (any("flagged" in tag for tag in self.tags))
 
 
     def to_dict(self):
@@ -185,9 +185,9 @@ class EvalResult:
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-            AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
-            AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
-            AutoEvalColumn.flagged.name: self.flagged,
+            AutoEvalColumn.merged.name: not( "merge" in self.tags if self.tags else False),
+            AutoEvalColumn.moe.name: not ( ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower()) ,
+            AutoEvalColumn.not_flagged.name: self.not_flagged,
         }
 
         for task in Tasks:

src/submission/check_validity.py

@@ -170,7 +170,6 @@ def get_model_tags(model_card, model: str):
     is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in ["moe", "mixtral"])
     # Hardcoding because of gating problem
     if "Qwen/Qwen1.5-32B" in model:
-        print("HERE NSHJNKJSNJLAS")
         is_moe_from_model_card = False
     is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-")
     if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:

src/tools/plots.py

@@ -34,7 +34,7 @@ def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
             # We ignore models that are flagged/no longer on the hub/not finished
             to_ignore = (
                 not row["still_on_hub"]
-                or row["flagged"]
+                or not row["not_flagged"]
                 or current_model in FLAGGED_MODELS
                 or row["status"] != "FINISHED"
             )
@@ -68,7 +68,6 @@ def create_plot_df(scores_df: dict[str : pd.DataFrame]) -> pd.DataFrame:
     """
     # Initialize the list to store DataFrames
     dfs = []
-
     # Iterate over the cols and create a new DataFrame for each column
     for col in BENCHMARK_COLS + [AutoEvalColumn.average.name]:
         d = scores_df[col].reset_index(drop=True)