eval.sh

#!/bin/bash

MODEL_PATH="/workspace/dolphin-2.6-mistral-7b-hf"
MODEL_NAME="dolphin-2.6-mistral-7b"
RESULTS_PATH="/workspace/results/$MODEL_NAME"
mkdir -p "$RESULTS_PATH"

PRETRAINED_ARGS="$MODEL_PATH,tensor_parallel_size=4,dtype=auto,trust_remote_code=True,gpu_memory_utilization=0.8"
MODEL_ARGS="pretrained=$PRETRAINED_ARGS"

tasks=(
    # "mmlu" 
    # "truthfulqa" 
    "gsm8k" 
    "hellaswag" 
    "arc_challenge" 
    "winogrande")

# Function to get the number of fewshot for a given task
get_num_fewshot() {
    case "$1" in
        "mmlu")          echo 5 ;;
        "truthfulqa")    echo 0 ;;
        "gsm8k")         echo 5 ;;
        "hellaswag")     echo 10 ;;
        "arc_challenge") echo 25 ;;
        "winogrande")    echo 5 ;;
        *)               echo 0 ;;
    esac
}

for TASK in "${tasks[@]}"; do
    echo lm_eval --model vllm --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
    lm_eval --model vllm --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
done


jq -s '[.[]]' $RESULTS_PATH/*.json > $RESULTS_PATH/eval_results.json

huggingface-cli upload cognitivecomputations/$MODEL_NAME $RESULTS_PATH/eval_results.json