codingutf8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license a single trainingtest example for simple sequence classification args guid unique id for the example texta string the untokenized text of the first sequence for single sequence tasks only this sequence must be specified textb optional string the untokenized text of the second sequence only must be specified for sequence pair tasks label optional string the label of the example this should be specified for train and dev examples but not for test examples serializes this instance to a json string return json dumpsdataclasses asdictself indent2 n dataclassfrozentrue class inputfeatures inputids listint attentionmask optionallistint none tokentypeids optionallistint none label optionalunionint float none def tojsonstringself base class for data converters for sequence classification data sets def getexamplefromtensordictself tensordict raise notimplementederror def gettrainexamplesself datadir gets a collection of inputexample for the dev set raise notimplementederror def gettestexamplesself datadir gets the list of labels for this data set raise notimplementederror def tfdsmapself example if lenself getlabels 1 example label self getlabelsintexample label return example classmethod def readtsvcls inputfile quotecharnone generic processor for a single sentence classification data set def initself labelsnone examplesnone modeclassification verbosefalse self labels if labels is none else labels self examples if examples is none else examples self mode mode self verbose verbose def lenself return lenself examples def getitemself idx if isinstanceidx slice return singlesentenceclassificationprocessorlabelsself labels examplesself examplesidx return self examplesidx classmethod def createfromcsv cls filename splitname columnlabel0 columntext1 columnidnone skipfirstrowfalse kwargs processor clskwargs processor addexamplesfromcsv filename splitnamesplitname columnlabelcolumnlabel columntextcolumntext columnidcolumnid skipfirstrowskipfirstrow overwritelabelstrue overwriteexamplestrue return processor classmethod def createfromexamplescls textsortextandlabels labelsnone kwargs processor clskwargs processor addexamplestextsortextandlabels labelslabels return processor def addexamplesfromcsv self filename splitname columnlabel0 columntext1 columnidnone skipfirstrowfalse overwritelabelsfalse overwriteexamplesfalse lines self readtsvfilename if skipfirstrow lines lines1 texts labels ids for i line in enumeratelines texts appendlinecolumntext labels appendlinecolumnlabel if columnid is not none ids appendlinecolumnid else guid fsplitnamei if splitname else stri ids appendguid return self addexamples texts labels ids overwritelabelsoverwritelabels overwriteexamplesoverwriteexamples def addexamples self textsortextandlabels labelsnone idsnone overwritelabelsfalse overwriteexamplesfalse if labels is not none and lentextsortextandlabels lenlabels raise valueerror ftext and labels have mismatched lengths lentextsortextandlabels and lenlabels if ids is not none and lentextsortextandlabels lenids raise valueerrorftext and ids have mismatched lengths lentextsortextandlabels and lenids if ids is none ids none lentextsortextandlabels if labels is none labels none lentextsortextandlabels examples addedlabels set for textortextandlabel label guid in ziptextsortextandlabels labels ids if isinstancetextortextandlabel tuple list and label is none text label textortextandlabel else text textortextandlabel addedlabels addlabel examples appendinputexampleguidguid textatext textbnone labellabel update examples if overwriteexamples self examples examples else self examples extendexamples update labels if overwritelabels self labels listaddedlabels else self labels listsetself labels unionaddedlabels return self examples def getfeatures self tokenizer maxlengthnone padonleftfalse padtoken0 maskpaddingwithzerotrue returntensorsnone if maxlength is none maxlength tokenizer maxlen labelmap label i for i label in enumerateself labels allinputids for exindex example in enumerateself examples if exindex 10000 0 logger infoftokenizing example exindex inputids tokenizer encode example texta addspecialtokenstrue maxlengthminmaxlength tokenizer maxlen allinputids appendinputids batchlength maxleninputids for inputids in allinputids features for exindex inputids example in enumeratezipallinputids self examples if exindex 10000 0 logger infofwriting example exindexlenself examples the mask has 1 for real tokens and 0 for padding tokens only real tokens are attended to attentionmask 1 if maskpaddingwithzero else 0 leninputids zeropad up to the sequence length paddinglength batchlength leninputids if padonleft inputids padtoken paddinglength inputids attentionmask 0 if maskpaddingwithzero else 1 paddinglength attentionmask else inputids inputids padtoken paddinglength attentionmask attentionmask 0 if maskpaddingwithzero else 1 paddinglength if leninputids batchlength raise valueerrorferror with input length leninputids vs batchlength if lenattentionmask batchlength raise valueerrorferror with input length lenattentionmask vs batchlength if self mode classification label labelmapexample label elif self mode regression label floatexample label else raise valueerrorself mode if exindex 5 and self verbose logger info example logger infofguid example guid logger infofinputids joinstrx for x in inputids logger infofattentionmask joinstrx for x in attentionmask logger infoflabel example label id label features appendinputfeaturesinputidsinputids attentionmaskattentionmask labellabel if returntensors is none return features elif returntensors tf if not istfavailable raise runtimeerrorreturntensors set to tf but tensorflow 2 0 can t be imported import tensorflow as tf def gen for ex in features yield inputids ex inputids attentionmask ex attentionmask ex label dataset tf data dataset fromgenerator gen inputids tf int32 attentionmask tf int32 tf int64 inputids tf tensorshapenone attentionmask tf tensorshapenone tf tensorshape return dataset elif returntensors pt if not istorchavailable raise runtimeerrorreturntensors set to pt but pytorch can t be imported import torch from torch utils data import tensordataset allinputids torch tensorf inputids for f in features dtypetorch long allattentionmask torch tensorf attentionmask for f in features dtypetorch long if self mode classification alllabels torch tensorf label for f in features dtypetorch long elif self mode regression alllabels torch tensorf label for f in features dtypetorch float dataset tensordatasetallinputids allattentionmask alllabels return dataset else raise valueerrorreturntensors should be one of tf or pt coding utf 8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license a single training test example for simple sequence classification args guid unique id for the example text_a string the untokenized text of the first sequence for single sequence tasks only this sequence must be specified text_b optional string the untokenized text of the second sequence only must be specified for sequence pair tasks label optional string the label of the example this should be specified for train and dev examples but not for test examples serializes this instance to a json string a single set of features of data property names are the same names as the corresponding inputs to a model args input_ids indices of input sequence tokens in the vocabulary attention_mask mask to avoid performing attention on padding token indices mask values selected in 0 1 usually 1 for tokens that are not masked 0 for masked padded tokens token_type_ids optional segment token indices to indicate first and second portions of the inputs only some models use them label optional label corresponding to the input int for classification problems float for regression problems serializes this instance to a json string base class for data converters for sequence classification data sets gets an example from a dict with tensorflow tensors args tensor_dict keys and values should match the corresponding glue tensorflow_dataset examples gets a collection of inputexample for the train set gets a collection of inputexample for the dev set gets a collection of inputexample for the test set gets the list of labels for this data set some tensorflow_datasets datasets are not formatted the same way the glue datasets are this method converts examples to the correct format reads a tab separated value file generic processor for a single sentence classification data set update examples update labels convert examples in a list of inputfeatures args tokenizer instance of a tokenizer that will tokenize the examples max_length maximum example length pad_on_left if set to true the examples will be padded on the left rather than on the right default pad_token padding token mask_padding_with_zero if set to true the attention mask will be filled by 1 for actual values and by 0 for padded values if set to false inverts it 1 for padded values 0 for actual values returns if the examples input is a tf data dataset will return a tf data dataset containing the task specific features if the input is a list of inputexamples will return a list of task specific inputfeatures which can be fed to the model the mask has 1 for real tokens and 0 for padding tokens only real tokens are attended to zero pad up to the sequence length
import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


logger = logging.get_logger(__name__)


@dataclass
class InputExample:
    

    guid: str
    text_a: str
    text_b: Optional[str] = None
    label: Optional[str] = None

    def to_json_string(self):
        
        return json.dumps(dataclasses.asdict(self), indent=2) + "\n"


@dataclass(frozen=True)
class InputFeatures:
    

    input_ids: List[int]
    attention_mask: Optional[List[int]] = None
    token_type_ids: Optional[List[int]] = None
    label: Optional[Union[int, float]] = None

    def to_json_string(self):
        
        return json.dumps(dataclasses.asdict(self)) + "\n"


class DataProcessor:
    

    def get_example_from_tensor_dict(self, tensor_dict):
        
        raise NotImplementedError()

    def get_train_examples(self, data_dir):
        
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        
        raise NotImplementedError()

    def get_test_examples(self, data_dir):
        
        raise NotImplementedError()

    def get_labels(self):
        
        raise NotImplementedError()

    def tfds_map(self, example):
        
        if len(self.get_labels()) > 1:
            example.label = self.get_labels()[int(example.label)]
        return example

    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        
        with open(input_file, "r", encoding="utf-8-sig") as f:
            return list(csv.reader(f, delimiter="\t", quotechar=quotechar))


class SingleSentenceClassificationProcessor(DataProcessor):
    

    def __init__(self, labels=None, examples=None, mode="classification", verbose=False):
        self.labels = [] if labels is None else labels
        self.examples = [] if examples is None else examples
        self.mode = mode
        self.verbose = verbose

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, idx):
        if isinstance(idx, slice):
            return SingleSentenceClassificationProcessor(labels=self.labels, examples=self.examples[idx])
        return self.examples[idx]

    @classmethod
    def create_from_csv(
        cls, file_name, split_name="", column_label=0, column_text=1, column_id=None, skip_first_row=False, **kwargs
    ):
        processor = cls(**kwargs)
        processor.add_examples_from_csv(
            file_name,
            split_name=split_name,
            column_label=column_label,
            column_text=column_text,
            column_id=column_id,
            skip_first_row=skip_first_row,
            overwrite_labels=True,
            overwrite_examples=True,
        )
        return processor

    @classmethod
    def create_from_examples(cls, texts_or_text_and_labels, labels=None, **kwargs):
        processor = cls(**kwargs)
        processor.add_examples(texts_or_text_and_labels, labels=labels)
        return processor

    def add_examples_from_csv(
        self,
        file_name,
        split_name="",
        column_label=0,
        column_text=1,
        column_id=None,
        skip_first_row=False,
        overwrite_labels=False,
        overwrite_examples=False,
    ):
        lines = self._read_tsv(file_name)
        if skip_first_row:
            lines = lines[1:]
        texts = []
        labels = []
        ids = []
        for i, line in enumerate(lines):
            texts.append(line[column_text])
            labels.append(line[column_label])
            if column_id is not None:
                ids.append(line[column_id])
            else:
                guid = f"{split_name}-{i}" if split_name else str(i)
                ids.append(guid)

        return self.add_examples(
            texts, labels, ids, overwrite_labels=overwrite_labels, overwrite_examples=overwrite_examples
        )

    def add_examples(
        self, texts_or_text_and_labels, labels=None, ids=None, overwrite_labels=False, overwrite_examples=False
    ):
        if labels is not None and len(texts_or_text_and_labels) != len(labels):
            raise ValueError(
                f"Text and labels have mismatched lengths {len(texts_or_text_and_labels)} and {len(labels)}"
            )
        if ids is not None and len(texts_or_text_and_labels) != len(ids):
            raise ValueError(f"Text and ids have mismatched lengths {len(texts_or_text_and_labels)} and {len(ids)}")
        if ids is None:
            ids = [None] * len(texts_or_text_and_labels)
        if labels is None:
            labels = [None] * len(texts_or_text_and_labels)
        examples = []
        added_labels = set()
        for text_or_text_and_label, label, guid in zip(texts_or_text_and_labels, labels, ids):
            if isinstance(text_or_text_and_label, (tuple, list)) and label is None:
                text, label = text_or_text_and_label
            else:
                text = text_or_text_and_label
            added_labels.add(label)
            examples.append(InputExample(guid=guid, text_a=text, text_b=None, label=label))

        
        if overwrite_examples:
            self.examples = examples
        else:
            self.examples.extend(examples)

        
        if overwrite_labels:
            self.labels = list(added_labels)
        else:
            self.labels = list(set(self.labels).union(added_labels))

        return self.examples

    def get_features(
        self,
        tokenizer,
        max_length=None,
        pad_on_left=False,
        pad_token=0,
        mask_padding_with_zero=True,
        return_tensors=None,
    ):
        
        if max_length is None:
            max_length = tokenizer.max_len

        label_map = {label: i for i, label in enumerate(self.labels)}

        all_input_ids = []
        for ex_index, example in enumerate(self.examples):
            if ex_index % 10000 == 0:
                logger.info(f"Tokenizing example {ex_index}")

            input_ids = tokenizer.encode(
                example.text_a,
                add_special_tokens=True,
                max_length=min(max_length, tokenizer.max_len),
            )
            all_input_ids.append(input_ids)

        batch_length = max(len(input_ids) for input_ids in all_input_ids)

        features = []
        for ex_index, (input_ids, example) in enumerate(zip(all_input_ids, self.examples)):
            if ex_index % 10000 == 0:
                logger.info(f"Writing example {ex_index}/{len(self.examples)}")
            
            
            attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)

            
            padding_length = batch_length - len(input_ids)
            if pad_on_left:
                input_ids = ([pad_token] * padding_length) + input_ids
                attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
            else:
                input_ids = input_ids + ([pad_token] * padding_length)
                attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

            if len(input_ids) != batch_length:
                raise ValueError(f"Error with input length {len(input_ids)} vs {batch_length}")
            if len(attention_mask) != batch_length:
                raise ValueError(f"Error with input length {len(attention_mask)} vs {batch_length}")

            if self.mode == "classification":
                label = label_map[example.label]
            elif self.mode == "regression":
                label = float(example.label)
            else:
                raise ValueError(self.mode)

            if ex_index < 5 and self.verbose:
                logger.info("*** Example ***")
                logger.info(f"guid: {example.guid}")
                logger.info(f"input_ids: {' '.join([str(x) for x in input_ids])}")
                logger.info(f"attention_mask: {' '.join([str(x) for x in attention_mask])}")
                logger.info(f"label: {example.label} (id = {label})")

            features.append(InputFeatures(input_ids=input_ids, attention_mask=attention_mask, label=label))

        if return_tensors is None:
            return features
        elif return_tensors == "tf":
            if not is_tf_available():
                raise RuntimeError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported")
            import tensorflow as tf

            def gen():
                for ex in features:
                    yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

            dataset = tf.data.Dataset.from_generator(
                gen,
                ({"input_ids": tf.int32, "attention_mask": tf.int32}, tf.int64),
                ({"input_ids": tf.TensorShape([None]), "attention_mask": tf.TensorShape([None])}, tf.TensorShape([])),
            )
            return dataset
        elif return_tensors == "pt":
            if not is_torch_available():
                raise RuntimeError("return_tensors set to 'pt' but PyTorch can't be imported")
            import torch
            from torch.utils.data import TensorDataset

            all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
            all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
            if self.mode == "classification":
                all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
            elif self.mode == "regression":
                all_labels = torch.tensor([f.label for f in features], dtype=torch.float)

            dataset = TensorDataset(all_input_ids, all_attention_mask, all_labels)
            return dataset
        else:
            raise ValueError("return_tensors should be one of 'tf' or 'pt'")