import pandas as pd
import json
import re
# Load the CSV file
leaderboard_df = []
with open("benchmark_results.csv", "r") as f:
header = f.readline().strip().split(",")
header = [h.strip() for h in header]
for i, line in enumerate(f):
leaderboard_df.append(line.strip().split(",", 13))
# Load metadata
metadata = json.load(open('metadata.json'))
for k, v in list(metadata.items()):
metadata[k.split(",")[0]] = v
# Create DataFrame
leaderboard_df = pd.DataFrame(leaderboard_df, columns=header)
# Filter and process DataFrame
leaderboard_df = leaderboard_df[(leaderboard_df["Benchmark Version"] == "eq-bench_v2_pl") | (
leaderboard_df["Benchmark Version"] == 'eq-bench_pl')]
leaderboard_df = leaderboard_df[["Model Path", "Benchmark Score", "Num Questions Parseable", "Error"]]
def parse_parseable(x):
if x["Num Questions Parseable"] == 'FAILED':
m = re.match(r'(\d+)\.0 questions were parseable', x["Error"])
return m.group(1)
return x["Num Questions Parseable"]
leaderboard_df["Num Questions Parseable"] = leaderboard_df[["Num Questions Parseable", "Error"]].apply(
lambda x: parse_parseable(x), axis=1)
NUMBER_OF_QUESTIONS = 171.0
def fraction_to_percentage(numerator: float, denominator: float) -> float:
return (numerator / denominator) * 100
leaderboard_df["Num Questions Parseable"] = leaderboard_df["Num Questions Parseable"].apply(lambda x: fraction_to_percentage(float(x), NUMBER_OF_QUESTIONS))
def get_params(model_name):
if model_name in metadata:
return metadata[model_name]
else:
print(model_name)
return None
leaderboard_df["Params"] = leaderboard_df["Model Path"].apply(lambda x: get_params(x))
leaderboard_df["Benchmark Score"] = leaderboard_df["Benchmark Score"].replace('FAILED', None)
leaderboard_df["Benchmark Score"] = leaderboard_df["Benchmark Score"].astype(float) * ((leaderboard_df["Num Questions Parseable"].astype(float) / 100))
leaderboard_df.loc[leaderboard_df["Benchmark Score"] < 0, "Benchmark Score"] = 0
leaderboard_df = leaderboard_df.sort_values(by=["Benchmark Score", "Num Questions Parseable"], ascending=[False, False])
leaderboard_df = leaderboard_df.rename(columns={"Model Path": "Model", "Num Questions Parseable": "Percentage Questions Parseable"})
# Generate HTML with DataTables
html = """
Leaderboard
Leaderboard
Model
Params
Benchmark Score
Percentage Questions Parseable
Error
"""
# Add rows to the HTML table
for _, row in leaderboard_df.iterrows():
html += f"""
{row['Model']}
{row['Params']}
{row['Benchmark Score']:.2f}
{row['Percentage Questions Parseable']:.2f}
{row['Error']}
"""
# Close the HTML tags
html += """
"""
# Save the HTML to a file
with open("leaderboard.html", "w") as file:
file.write(html)
print("HTML leaderboard generated and saved as leaderboard.html")