codingutf8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xnli utils dataset loading and evaluation import os from utils import logging from utils import dataprocessor inputexample logger logging getloggername class xnliprocessordataprocessor def initself language trainlanguagenone self language language self trainlanguage trainlanguage def gettrainexamplesself datadir see base class lines self readtsvos path joindatadir xnli1 0xnli test tsv examples for i line in enumeratelines if i 0 continue language line0 if language self language continue guid ftesti texta line6 textb line7 label line1 if not isinstancetexta str raise valueerrorftraining input texta is not a string if not isinstancetextb str raise valueerrorftraining input textb is not a string if not isinstancelabel str raise valueerrorftraining label label is not a string examples appendinputexampleguidguid textatexta textbtextb labellabel return examples def getlabelsself coding utf 8 2018 the google ai language team s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xnli utils dataset loading and evaluation processor for the xnli dataset adapted from https github com google research bert blob f39e881b169b9d53bea03d2d341b31707a6c052b run_classifier py l207 see base class see base class see base class
import os

from ...utils import logging
from .utils import DataProcessor, InputExample


logger = logging.get_logger(__name__)


class XnliProcessor(DataProcessor):
    

    def __init__(self, language, train_language=None):
        self.language = language
        self.train_language = train_language

    def get_train_examples(self, data_dir):
        
        lg = self.language if self.train_language is None else self.train_language
        lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv"))
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            guid = f"train-{i}"
            text_a = line[0]
            text_b = line[1]
            label = "contradiction" if line[2] == "contradictory" else line[2]
            if not isinstance(text_a, str):
                raise ValueError(f"Training input {text_a} is not a string")
            if not isinstance(text_b, str):
                raise ValueError(f"Training input {text_b} is not a string")
            if not isinstance(label, str):
                raise ValueError(f"Training label {label} is not a string")
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

    def get_test_examples(self, data_dir):
        
        lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv"))
        examples = []
        for i, line in enumerate(lines):
            if i == 0:
                continue
            language = line[0]
            if language != self.language:
                continue
            guid = f"test-{i}"
            text_a = line[6]
            text_b = line[7]
            label = line[1]
            if not isinstance(text_a, str):
                raise ValueError(f"Training input {text_a} is not a string")
            if not isinstance(text_b, str):
                raise ValueError(f"Training input {text_b} is not a string")
            if not isinstance(label, str):
                raise ValueError(f"Training label {label} is not a string")
            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

    def get_labels(self):
        
        return ["contradiction", "entailment", "neutral"]


xnli_processors = {
    "xnli": XnliProcessor,
}

xnli_output_modes = {
    "xnli": "classification",
}

xnli_tasks_num_labels = {
    "xnli": 3,
}