File size: 5,210 Bytes
b98c534 6634f62 b98c534 9961873 6634f62 b98c534 9961873 b98c534 650da80 6634f62 650da80 b98c534 650da80 6634f62 b98c534 650da80 6634f62 650da80 b98c534 650da80 b98c534 650da80 6634f62 650da80 b98c534 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
{
"name": "Hugging Face LLM Comparisons",
"schema_version": "latest",
"metadata": {
"parameters": {
"CoLA_ex_prompt": "Is the sentence grammatical or ungrammatical?\n\n\"This building is than that one.\"",
"SST_2_ex_prompt": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
"WNLI_ex_prompt": "Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun?\n\n\"A) Lily spoke to Donna, breaking her concentration.\nB) Lily spoke to Donna, breaking Lily's concentration.\""
},
"models": {},
"default_model": null,
"model_parsers": null
},
"description": "**In this notebook, we compare the individual performance of HF hosted LLMs () on a few example questions from the GLUE benchmarks (https://gluebenchmark.com/tasks).**\n\n**Example questions taken from \"What is the GLUE Benchmark\" medium post - https://angelina-yang.medium.com/what-is-the-glue-benchmark-for-nlu-systems-61127b3cab3f**\n\n---\n\n| General Language Understanding Evaluation (GLUE) Tasks | Example Question |\n| ----------- | ----------- |\n| Corpus of Linguistic Acceptability (CoLA) | Is the sentence grammatical or ungrammatical? \"This building is than that one.\" |\n| Stanford Sentiment Treebank (SST) | Is the movie review positive, negative, or neutral? \"The movie is funny, smart, visually inventive, and most of all, alive.\" |\n| Winograd NLI (WNLI) | Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun? \"A) Lily spoke to Donna, breaking her concentration. B) Lily spoke to Donna, breaking Lily's concentration.\" |",
"prompts": [
{
"name": "mistral_7b_instruct_v0.1",
"input": "Is the movie review positive, negative, or neutral?\n\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
"metadata": {
"model": {
"name": "Text Generation",
"settings": {
"model": "mistralai/Mistral-7B-Instruct-v0.1"
}
},
"tags": null,
"parameters": {}
},
"outputs": [
{
"output_type": "execute_result",
"execution_count": 0,
"data": "\n\nThe movie review is positive.</s>",
"mime_type": null,
"metadata": {}
}
]
},
{
"name": "google_flan_t5_sm",
"input": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
"metadata": {
"model": {
"name": "Conversational",
"settings": {
"model": "google/flan-t5-small",
"max_new_tokens": 250,
"stream": false
}
},
"tags": null,
"parameters": {}
},
"outputs": [
{
"output_type": "execute_result",
"execution_count": 0,
"data": "positive",
"mime_type": null,
"metadata": {
"raw_response": {
"generated_text": "positive",
"conversation": {
"generated_responses": [
"positive"
],
"past_user_inputs": [
"Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\""
]
},
"warnings": [
"\nNo chat template is defined for this tokenizer - using a default chat template that implements the ChatML format (without BOS/EOS tokens!). If the default is not appropriate for your model, please set `tokenizer.chat_template` to an appropriate template. See https://huggingface.co./docs/transformers/main/chat_templating for more information.\n"
]
}
}
}
]
},
{
"name": "tinyllama-1_1B",
"input": "<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\n{{SST_2_ex_prompt}}</s>\n<|assistant|>",
"metadata": {
"model": {
"name": "Conversational",
"settings": {
"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
}
},
"tags": null,
"parameters": {}
},
"outputs": [
{
"output_type": "execute_result",
"execution_count": 0,
"data": "The movie review is positive.",
"mime_type": null,
"metadata": {
"raw_response": {
"generated_text": "The movie review is positive.",
"conversation": {
"generated_responses": [
"The movie review is positive."
],
"past_user_inputs": [
"<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\nIs the movie review positive, negative, or neutral?\n\n"The movie is funny, smart, visually inventive, and most of all, alive."</s>\n<|assistant|>"
]
}
}
}
}
]
}
]
} |