|
|
|
|
|
|
|
|
|
|
|
import unicodedata |
|
|
|
from fairseq.dataclass import ChoiceEnum |
|
|
|
|
|
class EvaluationTokenizer(object): |
|
"""A generic evaluation-time tokenizer, which leverages built-in tokenizers |
|
in sacreBLEU (https://github.com/mjpost/sacrebleu). It additionally provides |
|
lowercasing, punctuation removal and character tokenization, which are |
|
applied after sacreBLEU tokenization. |
|
|
|
Args: |
|
tokenizer_type (str): the type of sacreBLEU tokenizer to apply. |
|
lowercase (bool): lowercase the text. |
|
punctuation_removal (bool): remove punctuation (based on unicode |
|
category) from text. |
|
character_tokenization (bool): tokenize the text to characters. |
|
""" |
|
|
|
SPACE = chr(32) |
|
SPACE_ESCAPE = chr(9601) |
|
ALL_TOKENIZER_TYPES = ChoiceEnum(["none", "13a", "intl", "zh", "ja-mecab"]) |
|
|
|
def __init__( |
|
self, |
|
tokenizer_type: str = "13a", |
|
lowercase: bool = False, |
|
punctuation_removal: bool = False, |
|
character_tokenization: bool = False, |
|
): |
|
from sacrebleu.tokenizers import TOKENIZERS |
|
|
|
assert tokenizer_type in TOKENIZERS, f"{tokenizer_type}, {TOKENIZERS}" |
|
self.lowercase = lowercase |
|
self.punctuation_removal = punctuation_removal |
|
self.character_tokenization = character_tokenization |
|
self.tokenizer = TOKENIZERS[tokenizer_type] |
|
|
|
@classmethod |
|
def remove_punctuation(cls, sent: str): |
|
"""Remove punctuation based on Unicode category.""" |
|
return cls.SPACE.join( |
|
t |
|
for t in sent.split(cls.SPACE) |
|
if not all(unicodedata.category(c)[0] == "P" for c in t) |
|
) |
|
|
|
def tokenize(self, sent: str): |
|
tokenized = self.tokenizer()(sent) |
|
|
|
if self.punctuation_removal: |
|
tokenized = self.remove_punctuation(tokenized) |
|
|
|
if self.character_tokenization: |
|
tokenized = self.SPACE.join( |
|
list(tokenized.replace(self.SPACE, self.SPACE_ESCAPE)) |
|
) |
|
|
|
if self.lowercase: |
|
tokenized = tokenized.lower() |
|
|
|
return tokenized |
|
|
