Update

metrics.py

@@ -1,21 +1,28 @@
 import re
 import duckdb
 import textwrap
+from typing import List, Tuple
 
-def _parse_answer(text: str) -> str:
+def _parse_answer(text: str) -> List[List[str]]:
     """
-    Converts text to lowercase. Interprets "," and "-->" as separators for
-    elements of a set. Within each set, drops all non-alphanumeric characters
+    Converts text to lowercase. Then interprets ";" as a separator between
+    alternatives. Within each alternative, interprets "," and "-->" as separators
+    for elements of a set. Within each set, drops all non-alphanumeric characters
     and returns that set.
 
+
     Another way to describe this is that we interpret adjacent words as
     phrases that must be present literally. However, comma and arrow separate
     distinct phrases that may be present in any order. All other characters
     are dropped.
     """
     text = text.lower()
-    groups = re.split(r'-->|,', text)
-    return [" ".join(re.findall(r'\b\w+\b', group)) for group in groups]
+    alternatives = re.split(r';', text)
+    result = [ ]
+    for alternative in alternatives:
+        groups = re.split(r'-->|,', alternative)
+        result.append([" ".join(re.findall(r'\b\w+\b', group)) for group in groups])
+    return result
 
 def _answer_without_thoughts(completion: str) -> str:
     if "<think>" not in completion[:200]:
@@ -33,9 +40,11 @@ def _check_answer(completion: str, answer: str) -> bool:
     completion. We ignore "thoughts", capitalization, and punctuation.
     """
     completion = _answer_without_thoughts(completion).lower()
-    answer_phrases = _parse_answer(answer)
-    r = all(phrase in completion for phrase in answer_phrases)
-    return r
+    alternative_answers = _parse_answer(answer)
+    for answer_phrases in alternative_answers:
+        if all(phrase in completion for phrase in answer_phrases):
+            return True
+    return False
 
 
 def _clip_text(text: str, width: int) -> str:

results.duckdb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa7c7911a1ecf7fe4223995e3d393dd78cf8d4023409197854bf471fd8ab7c48
-size 32518144
+oid sha256:c7d3a9c4a997e0b6741249ba973be3c145b6f660381633d35e5eaa94353ea30f
+size 39333888