codingutf8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license glue processors and helpers import os import warnings from dataclasses import asdict from enum import enum from typing import list optional union from tokenizationutils import pretrainedtokenizer from utils import istfavailable logging from utils import dataprocessor inputexample inputfeatures if istfavailable import tensorflow as tf logger logging getloggername deprecationwarning this 0 will be removed from the library soon preprocessing should be handled with the datasets library you can have a look at this example script for pointers https github comhuggingfacetransformersblobmainexamplespytorchtextclassificationrunglue py def glueconvertexamplestofeatures examples unionlistinputexample tf data dataset tokenizer pretrainedtokenizer maxlength optionalint none tasknone labellistnone outputmodenone warnings warndeprecationwarning formatfunction futurewarning if istfavailable and isinstanceexamples tf data dataset if task is none raise valueerrorwhen calling glueconvertexamplestofeatures from tf the task parameter is required return tfglueconvertexamplestofeaturesexamples tokenizer maxlengthmaxlength tasktask return glueconvertexamplestofeatures examples tokenizer maxlengthmaxlength tasktask labellistlabellist outputmodeoutputmode if istfavailable def tfglueconvertexamplestofeatures examples tf data dataset tokenizer pretrainedtokenizer taskstr maxlength optionalint none tf data dataset processor glueprocessorstask examples processor tfdsmapprocessor getexamplefromtensordictexample for example in examples features glueconvertexamplestofeaturesexamples tokenizer maxlengthmaxlength tasktask labeltype tf float32 if task stsb else tf int64 def gen for ex in features d k v for k v in asdictex items if v is not none label d poplabel yield d label inputnames tokenizer modelinputnames return tf data dataset fromgenerator gen k tf int32 for k in inputnames labeltype k tf tensorshapenone for k in inputnames tf tensorshape def glueconvertexamplestofeatures examples listinputexample tokenizer pretrainedtokenizer maxlength optionalint none tasknone labellistnone outputmodenone if maxlength is none maxlength tokenizer modelmaxlength if task is not none processor glueprocessorstask if labellist is none labellist processor getlabels logger infofusing label list labellist for task task if outputmode is none outputmode glueoutputmodestask logger infofusing output mode outputmode for task task labelmap label i for i label in enumeratelabellist def labelfromexampleexample inputexample unionint float none if example label is none return none if outputmode classification return labelmapexample label elif outputmode regression return floatexample label raise keyerroroutputmode labels labelfromexampleexample for example in examples batchencoding tokenizer example texta example textb for example in examples maxlengthmaxlength paddingmaxlength truncationtrue features for i in rangelenexamples inputs k batchencodingki for k in batchencoding feature inputfeaturesinputs labellabelsi features appendfeature for i example in enumerateexamples 5 logger info example logger infofguid example guid logger infoffeatures featuresi return features class outputmodeenum classification classification regression regression class mrpcprocessordataprocessor see base class return inputexample tensordictidx numpy tensordictsentence1 numpy decodeutf8 tensordictsentence2 numpy decodeutf8 strtensordictlabel numpy def gettrainexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir dev tsv dev def gettestexamplesself datadir see base class return 0 1 def createexamplesself lines settype processor for the multinli data set glue version def initself args kwargs super initargs kwargs warnings warndeprecationwarning formatprocessor futurewarning def getexamplefromtensordictself tensordict see base class return self createexamplesself readtsvos path joindatadir train tsv train def getdevexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir testmatched tsv testmatched def getlabelsself creates examples for the training dev and test sets examples for i line in enumeratelines if i 0 continue guid fsettypeline0 texta line8 textb line9 label none if settype startswithtest else line1 examples appendinputexampleguidguid textatexta textbtextb labellabel return examples class mnlimismatchedprocessormnliprocessor see base class return self createexamplesself readtsvos path joindatadir devmismatched tsv devmismatched def gettestexamplesself datadir processor for the cola data set glue version def initself args kwargs super initargs kwargs warnings warndeprecationwarning formatprocessor futurewarning def getexamplefromtensordictself tensordict see base class return self createexamplesself readtsvos path joindatadir train tsv train def getdevexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir test tsv test def getlabelsself creates examples for the training dev and test sets testmode settype test if testmode lines lines1 textindex 1 if testmode else 3 examples for i line in enumeratelines guid fsettypei texta linetextindex label none if testmode else line1 examples appendinputexampleguidguid textatexta textbnone labellabel return examples class sst2processordataprocessor see base class return inputexample tensordictidx numpy tensordictsentence numpy decodeutf8 none strtensordictlabel numpy def gettrainexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir dev tsv dev def gettestexamplesself datadir see base class return 0 1 def createexamplesself lines settype processor for the stsb data set glue version def initself args kwargs super initargs kwargs warnings warndeprecationwarning formatprocessor futurewarning def getexamplefromtensordictself tensordict see base class return self createexamplesself readtsvos path joindatadir train tsv train def getdevexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir test tsv test def getlabelsself creates examples for the training dev and test sets examples for i line in enumeratelines if i 0 continue guid fsettypeline0 texta line7 textb line8 label none if settype test else line1 examples appendinputexampleguidguid textatexta textbtextb labellabel return examples class qqpprocessordataprocessor see base class return inputexample tensordictidx numpy tensordictquestion1 numpy decodeutf8 tensordictquestion2 numpy decodeutf8 strtensordictlabel numpy def gettrainexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir dev tsv dev def gettestexamplesself datadir see base class return 0 1 def createexamplesself lines settype processor for the qnli data set glue version def initself args kwargs super initargs kwargs warnings warndeprecationwarning formatprocessor futurewarning def getexamplefromtensordictself tensordict see base class return self createexamplesself readtsvos path joindatadir train tsv train def getdevexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir test tsv test def getlabelsself creates examples for the training dev and test sets examples for i line in enumeratelines if i 0 continue guid fsettypeline0 texta line1 textb line2 label none if settype test else line1 examples appendinputexampleguidguid textatexta textbtextb labellabel return examples class rteprocessordataprocessor see base class return inputexample tensordictidx numpy tensordictsentence1 numpy decodeutf8 tensordictsentence2 numpy decodeutf8 strtensordictlabel numpy def gettrainexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir dev tsv dev def gettestexamplesself datadir see base class return entailment notentailment def createexamplesself lines settype processor for the wnli data set glue version def initself args kwargs super initargs kwargs warnings warndeprecationwarning formatprocessor futurewarning def getexamplefromtensordictself tensordict see base class return self createexamplesself readtsvos path joindatadir train tsv train def getdevexamplesself datadir see base class return self createexamplesself readtsvos path joindatadir test tsv test def getlabelsself creates examples for the training dev and test sets examples for i line in enumeratelines if i 0 continue guid fsettypeline0 texta line1 textb line2 label none if settype test else line1 examples appendinputexampleguidguid textatexta textbtextb labellabel return examples gluetasksnumlabels cola 2 mnli 3 mrpc 2 sst2 2 stsb 1 qqp 2 qnli 2 rte 2 wnli 2 glueprocessors cola colaprocessor mnli mnliprocessor mnlimm mnlimismatchedprocessor mrpc mrpcprocessor sst2 sst2processor stsb stsbprocessor qqp qqpprocessor qnli qnliprocessor rte rteprocessor wnli wnliprocessor glueoutputmodes cola classification mnli classification mnlimm classification mrpc classification sst2 classification stsb regression qqp classification qnli classification rte classification wnli classification coding utf 8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license glue processors and helpers loads a data file into a list of inputfeatures args examples list of inputexamples or tf data dataset containing the examples tokenizer instance of a tokenizer that will tokenize the examples max_length maximum example length defaults to the tokenizer s max_len task glue task label_list list of labels can be obtained from the processor using the processor get_labels method output_mode string indicating the output mode either regression or classification returns if the examples input is a tf data dataset will return a tf data dataset containing the task specific features if the input is a list of inputexamples will return a list of task specific inputfeatures which can be fed to the model returns a tf data dataset containing the task specific features processor for the mrpc data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the multinli data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the multinli mismatched data set glue version see base class see base class processor for the cola data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the sst 2 data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the sts b data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the qqp data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the qnli data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the rte data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets processor for the wnli data set glue version see base class see base class see base class see base class see base class creates examples for the training dev and test sets
import os
import warnings
from dataclasses import asdict
from enum import Enum
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_tf_available, logging
from .utils import DataProcessor, InputExample, InputFeatures


if is_tf_available():
    import tensorflow as tf

logger = logging.get_logger(__name__)

DEPRECATION_WARNING = (
    "This {0} will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets "
    "library. You can have a look at this example script for pointers: "
    "https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py"
)


def glue_convert_examples_to_features(
    examples: Union[List[InputExample], "tf.data.Dataset"],
    tokenizer: PreTrainedTokenizer,
    max_length: Optional[int] = None,
    task=None,
    label_list=None,
    output_mode=None,
):
    
    warnings.warn(DEPRECATION_WARNING.format("function"), FutureWarning)
    if is_tf_available() and isinstance(examples, tf.data.Dataset):
        if task is None:
            raise ValueError("When calling glue_convert_examples_to_features from TF, the task parameter is required.")
        return _tf_glue_convert_examples_to_features(examples, tokenizer, max_length=max_length, task=task)
    return _glue_convert_examples_to_features(
        examples, tokenizer, max_length=max_length, task=task, label_list=label_list, output_mode=output_mode
    )


if is_tf_available():

    def _tf_glue_convert_examples_to_features(
        examples: tf.data.Dataset,
        tokenizer: PreTrainedTokenizer,
        task=str,
        max_length: Optional[int] = None,
    ) -> tf.data.Dataset:
        
        processor = glue_processors[task]()
        examples = [processor.tfds_map(processor.get_example_from_tensor_dict(example)) for example in examples]
        features = glue_convert_examples_to_features(examples, tokenizer, max_length=max_length, task=task)
        label_type = tf.float32 if task == "sts-b" else tf.int64

        def gen():
            for ex in features:
                d = {k: v for k, v in asdict(ex).items() if v is not None}
                label = d.pop("label")
                yield (d, label)

        input_names = tokenizer.model_input_names

        return tf.data.Dataset.from_generator(
            gen,
            ({k: tf.int32 for k in input_names}, label_type),
            ({k: tf.TensorShape([None]) for k in input_names}, tf.TensorShape([])),
        )


def _glue_convert_examples_to_features(
    examples: List[InputExample],
    tokenizer: PreTrainedTokenizer,
    max_length: Optional[int] = None,
    task=None,
    label_list=None,
    output_mode=None,
):
    if max_length is None:
        max_length = tokenizer.model_max_length

    if task is not None:
        processor = glue_processors[task]()
        if label_list is None:
            label_list = processor.get_labels()
            logger.info(f"Using label list {label_list} for task {task}")
        if output_mode is None:
            output_mode = glue_output_modes[task]
            logger.info(f"Using output mode {output_mode} for task {task}")

    label_map = {label: i for i, label in enumerate(label_list)}

    def label_from_example(example: InputExample) -> Union[int, float, None]:
        if example.label is None:
            return None
        if output_mode == "classification":
            return label_map[example.label]
        elif output_mode == "regression":
            return float(example.label)
        raise KeyError(output_mode)

    labels = [label_from_example(example) for example in examples]

    batch_encoding = tokenizer(
        [(example.text_a, example.text_b) for example in examples],
        max_length=max_length,
        padding="max_length",
        truncation=True,
    )

    features = []
    for i in range(len(examples)):
        inputs = {k: batch_encoding[k][i] for k in batch_encoding}

        feature = InputFeatures(**inputs, label=labels[i])
        features.append(feature)

    for i, example in enumerate(examples[:5]):
        logger.info("*** Example ***")
        logger.info(f"guid: {example.guid}")
        logger.info(f"features: {features[i]}")

    return features


class OutputMode(Enum):
    classification = "classification"
    regression = "regression"


class MrpcProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence1"].numpy().decode("utf-8"),
            tensor_dict["sentence2"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        logger.info(f"LOOKING AT {os.path.join(data_dir, 'train.tsv')}")
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{i}"
            text_a = line[3]
            text_b = line[4]
            label = None if set_type == "test" else line[0]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class MnliProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["premise"].numpy().decode("utf-8"),
            tensor_dict["hypothesis"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), "dev_matched")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test_matched")

    def get_labels(self):
        
        return ["contradiction", "entailment", "neutral"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            text_a = line[8]
            text_b = line[9]
            label = None if set_type.startswith("test") else line[-1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class MnliMismatchedProcessor(MnliProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_mismatched.tsv")), "dev_mismatched")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test_mismatched.tsv")), "test_mismatched")


class ColaProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence"].numpy().decode("utf-8"),
            None,
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        
        test_mode = set_type == "test"
        if test_mode:
            lines = lines[1:]
        text_index = 1 if test_mode else 3
        examples = []
        for i, line in enumerate(lines):
            guid = f"{set_type}-{i}"
            text_a = line[text_index]
            label = None if test_mode else line[1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples


class Sst2Processor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence"].numpy().decode("utf-8"),
            None,
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        text_index = 1 if set_type == "test" else 0
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{i}"
            text_a = line[text_index]
            label = None if set_type == "test" else line[1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples


class StsbProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence1"].numpy().decode("utf-8"),
            tensor_dict["sentence2"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return [None]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            text_a = line[7]
            text_b = line[8]
            label = None if set_type == "test" else line[-1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class QqpProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["question1"].numpy().decode("utf-8"),
            tensor_dict["question2"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        
        test_mode = set_type == "test"
        q1_index = 1 if test_mode else 3
        q2_index = 2 if test_mode else 4
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            try:
                text_a = line[q1_index]
                text_b = line[q2_index]
                label = None if test_mode else line[5]
            except IndexError:
                continue
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class QnliProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["question"].numpy().decode("utf-8"),
            tensor_dict["sentence"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["entailment", "not_entailment"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            text_a = line[1]
            text_b = line[2]
            label = None if set_type == "test" else line[-1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class RteProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence1"].numpy().decode("utf-8"),
            tensor_dict["sentence2"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["entailment", "not_entailment"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            text_a = line[1]
            text_b = line[2]
            label = None if set_type == "test" else line[-1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


class WnliProcessor(DataProcessor):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        warnings.warn(DEPRECATION_WARNING.format("processor"), FutureWarning)

    def get_example_from_tensor_dict(self, tensor_dict):
        
        return InputExample(
            tensor_dict["idx"].numpy(),
            tensor_dict["sentence1"].numpy().decode("utf-8"),
            tensor_dict["sentence2"].numpy().decode("utf-8"),
            str(tensor_dict["label"].numpy()),
        )

    def get_train_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_test_examples(self, data_dir):
        
        return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"{set_type}-{line[0]}"
            text_a = line[1]
            text_b = line[2]
            label = None if set_type == "test" else line[-1]
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples


glue_tasks_num_labels = {
    "cola": 2,
    "mnli": 3,
    "mrpc": 2,
    "sst-2": 2,
    "sts-b": 1,
    "qqp": 2,
    "qnli": 2,
    "rte": 2,
    "wnli": 2,
}

glue_processors = {
    "cola": ColaProcessor,
    "mnli": MnliProcessor,
    "mnli-mm": MnliMismatchedProcessor,
    "mrpc": MrpcProcessor,
    "sst-2": Sst2Processor,
    "sts-b": StsbProcessor,
    "qqp": QqpProcessor,
    "qnli": QnliProcessor,
    "rte": RteProcessor,
    "wnli": WnliProcessor,
}

glue_output_modes = {
    "cola": "classification",
    "mnli": "classification",
    "mnli-mm": "classification",
    "mrpc": "classification",
    "sst-2": "classification",
    "sts-b": "regression",
    "qqp": "classification",
    "qnli": "classification",
    "rte": "classification",
    "wnli": "classification",
}