""" Source and credits: https://github.com/ZubinGou/math-evaluation-harness/blob/main/python_executor.py """ import re import regex import sympy from typing import TypeVar, Iterable, List, Union, Any, Dict from word2number import w2n from utils import * def lower_keys(example): new_example = {} for key, value in example.items(): if key != key.lower(): new_key = key.lower() new_example[new_key] = value else: new_example[key] = value return new_example def _fix_fracs(string): substrs = string.split("\\frac") new_str = substrs[0] if len(substrs) > 1: substrs = substrs[1:] for substr in substrs: new_str += "\\frac" if len(substr) > 0 and substr[0] == "{": new_str += substr else: try: assert len(substr) >= 2 except: return string a = substr[0] b = substr[1] if b != "{": if len(substr) > 2: post_substr = substr[2:] new_str += "{" + a + "}{" + b + "}" + post_substr else: new_str += "{" + a + "}{" + b + "}" else: if len(substr) > 2: post_substr = substr[2:] new_str += "{" + a + "}" + b + post_substr else: new_str += "{" + a + "}" + b string = new_str return string def _fix_a_slash_b(string): if len(string.split("/")) != 2: return string a = string.split("/")[0] b = string.split("/")[1] try: if "sqrt" not in a: a = int(a) if "sqrt" not in b: b = int(b) assert string == "{}/{}".format(a, b) new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" return new_string except: return string def _fix_sqrt(string): _string = re.sub(r"\\sqrt(\w+)", r"\\sqrt{\1}", string) return _string def convert_word_number(text:str) -> str: try: text = str(w2n.word_to_num(text)) except: pass return text # units mainly from MathQA unit_texts = [ "east", "degree", "mph", "kmph", "ft", "m sqaure", " m east", "sq m", "deg", "mile", "q .", "monkey", "prime", "ratio", "profit of rs", "rd", "o", "gm", "p . m", "lb", "tile", "per", "dm", "lt", "gain", "ab", "way", "west", "a .", "b .", "c .", "d .", "e .", "f .", "g .", "h .", "t", "a", "h", "no change", "men", "soldier", "pie", "bc", "excess", "st", "inches", "noon", "percent", "by", "gal", "kmh", "c", "acre", "rise", "a . m", "th", "π r 2", "sq", "mark", "l", "toy", "coin", "sq . m", "gallon", "° f", "profit", "minw", "yr", "women", "feet", "am", "pm", "hr", "cu cm", "square", "v â € ™", "are", "rupee", "rounds", "cubic", "cc", "mtr", "s", "ohm", "number", "kmph", "day", "hour", "minute", "min", "second", "man", "woman", "sec", "cube", "mt", "sq inch", "mp", "∏ cm ³", "hectare", "more", "sec", "unit", "cu . m", "cm 2", "rs .", "rs", "kg", "g", "month", "km", "m", "cm", "mm", "apple", "liter", "loss", "yard", "pure", "year", "increase", "decrease", "d", "less", "Surface", "litre", "pi sq m", "s .", "metre", "meter", "inch", ] unit_texts.extend([t + "s" for t in unit_texts]) def strip_string(string): string = str(string).strip() # linebreaks string = string.replace("\n", "") # right "." string = string.rstrip(".") # remove inverse spaces # replace \\ with \ string = string.replace("\\!", "") # string = string.replace("\\ ", "") # string = string.replace("\\\\", "\\") # matrix string = re.sub(r'\\begin\{array\}\{.*?\}', r'\\begin{pmatrix}', string) string = re.sub(r'\\end\{array\}', r'\\end{pmatrix}', string) string = string.replace("bmatrix", "pmatrix") # replace tfrac and dfrac with frac string = string.replace("tfrac", "frac") string = string.replace("dfrac", "frac") # remove \left and \right string = string.replace("\\left", "") string = string.replace("\\right", "") string = string.replace("\\{", "{") string = string.replace("\\}", "}") # Remove unit: miles, dollars if after is not none _string = re.sub(r"\\text{.*?}$", "", string).strip() if _string != "" and _string != string: # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) string = _string # Remove unit: texts for _ in range(2): for unit_text in unit_texts: # use regex, the prefix should be either the start of the string or a non-alphanumeric character # the suffix should be either the end of the string or a non-alphanumeric character _string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string) if _string != "": string = _string # Remove circ (degrees) string = string.replace("^{\\circ}", "") string = string.replace("^\\circ", "") # remove dollar signs string = string.replace("\\$", "") string = string.replace("$", "") # convert word number to digit string = convert_word_number(string) # replace "\\text{...}" to "..." string = re.sub(r"\\text\{(.*?)\}", r"\1", string) for key in ['x=', 'y=', 'z=', 'x\\in', 'y\\in', 'z\\in', 'x\\to', 'y\\to', 'z\\to']: string = string.replace(key, "") string = string.replace("\\emptyset", r"{}") string = string.replace("(-\\infty,\\infty)", "\\mathbb{R}") # remove percentage string = string.replace("\\%", "") string = string.replace("\%", "") string = string.replace("%", "") # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string string = string.replace(" .", " 0.") string = string.replace("{.", "{0.") # cdot # string = string.replace("\\cdot", "") if string.startswith("{") and string.endswith("}") and string.isalnum() or \ string.startswith("(") and string.endswith(")") and string.isalnum() or \ string.startswith("[") and string.endswith("]") and string.isalnum(): string = string[1:-1] # inf string = string.replace("infinity", "\\infty") if "\\infty" not in string: string = string.replace("inf", "\\infty") string = string.replace("+\\inity", "\\infty") # and string = string.replace("and", "") string = string.replace("\\mathbf", "") # use regex to remove \mbox{...} string = re.sub(r"\\mbox{.*?}", "", string) # quote string.replace("'", "") string.replace("\"", "") # i, j if "j" in string and "i" not in string: string = string.replace("j", "i") # replace a.000b where b is not number or b is end, with ab, use regex string = re.sub(r"(\d+)\.0*([^\d])", r"\1\2", string) string = re.sub(r"(\d+)\.0*$", r"\1", string) # if empty, return empty string if len(string) == 0: return string if string[0] == ".": string = "0" + string # to consider: get rid of e.g. "k = " or "q = " at beginning if len(string.split("=")) == 2: if len(string.split("=")[0]) <= 2: string = string.split("=")[1] string = _fix_sqrt(string) string = string.replace(" ", "") # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} string = _fix_fracs(string) # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y string = _fix_a_slash_b(string) return string def extract_multi_choice_answer(pred_str): # TODO: SFT models if 'Problem:' in pred_str: pred_str = pred_str.split("Problem:", 1)[0] pred_str = pred_str.replace("choice is", "answer is") patt = regex.search(r"answer is \(?(?P[abcde])\)?", pred_str.lower()) if patt is not None: return patt.group('ans').upper() return 'placeholder' def extract_answer(pred_str, data_name): if data_name in ["mmlu_stem", "sat_math", "mathqa"]: return extract_multi_choice_answer(pred_str) if 'final answer is $' in pred_str and '$. I hope' in pred_str: # minerva_math tmp = pred_str.split('final answer is $', 1)[1] pred = tmp.split('$. I hope', 1)[0].strip() elif 'boxed' in pred_str: ans = pred_str.split('boxed')[-1] if len(ans) == 0: return "" elif ans[0] == '{': stack = 1 a = '' for c in ans[1:]: if (c == '{'): stack += 1 a += c elif (c == '}'): stack -= 1 if (stack == 0): break a += c else: a += c else: a = ans.split('$')[0].strip() pred = a elif ('he answer is' in pred_str): pred = pred_str.split('he answer is')[-1].strip() elif ('final answer is' in pred_str): pred = pred_str.split('final answer is')[-1].strip() # elif extract_program_output(pred_str) != "": # fall back to program # pred = extract_program_output(pred_str) else: # use the last number pattern = '-?\d*\.?\d+' pred = re.findall(pattern, pred_str.replace(",", "")) if(len(pred) >= 1): pred = pred[-1] else: pred = '' # multiple line # pred = pred.split("\n")[0] pred = re.sub(r"\n\s*", "", pred) if pred != "" and pred[0] == ":": pred = pred[1:] if pred != "" and pred[-1] == ".": pred = pred[:-1] if pred != "" and pred[-1] == "/": pred = pred[:-1] pred = strip_string(pred) return pred def parse_ground_truth(example: Dict[str, Any], data_name): # parse ground truth if data_name in ["MATH", "math", "math_oai", "minerva_math", "ocw", "amps", "hungarian_exam"]: gt_ans = example['answer'] elif data_name == "gsm8k": gt_ans = example['answer'].split("####")[-1] elif data_name == "mmlu_stem": abcd = 'ABCD' gt_ans = abcd[example['answer']] elif data_name == "gpqa": gt_ans = example['correct answer'] else: raise NotImplementedError(f"`{data_name}`") # post process gt_ans = strip_string(gt_ans) return gt_ans def parse_question(example, data_name): question = "" if data_name == "mmlu_stem": options = example['choices'] assert len(options) == 4 for i, (label, option) in enumerate(zip('ABCD', options)): options[i] = f"({label}) {str(option).strip()}" options = ", ".join(options) question = f"{example['question'].strip()}\nWhat of the following is the right choice? Explain your answer.\n{options}" else: for key in ['question', 'problem', 'Question', 'input']: if key in example: question = example[key] break assert question != "" # Yes or No question gt_ans = parse_ground_truth(example, data_name) gt_lower = gt_ans.lower() if gt_lower in ["true", "false"]: question += " (True or False)" if gt_lower in ["yes", "no"]: question += " (Yes or No)" return question.strip() def _test_extract_answer(): text= """ The answer is $\\boxed{\left( \\begin{array}{ccc} -13 & 4 & -2 \\\\ 7 & 8 & -3 \\\\ 0 & 18 & -7 \\\\ 6 & 12 & 5 \\\\ \\end{array} \\right)}$. """ print(extract_answer(text, "math")) # should output a dict if __name__ == "__main__": _test_extract_answer()