<jupyter_start><jupyter_text>RTE : Recognizing Textual EntailmentThe Recognizing Textual Entailment(RTE) task is a sentence pair classification task. It consists of sentence pairs drawn from annual text data challenge sets and annotated for textual entailment.See [website](https://aclweb.org/aclwiki/Recognizing_Textual_Entailment) for more info.<jupyter_code>import numpy as np
import pandas as pd
import os
import sys
import csv
from sklearn import metrics
from sklearn.metrics import classification_report

sys.path.append("../") 
from bert_sklearn import BertClassifier

DATADIR = os.getcwd() + '/glue_data'
%%bash
python3 download_glue_data.py --data_dir glue_data --tasks RTE 
"""
RTE train data size: 2489 
RTE dev data size: 277 
"""

def read_tsv(filename,quotechar=None):
    with open(filename, "r", encoding='utf-8') as f:
        return list(csv.reader(f,delimiter="\t",quotechar=quotechar))
    

def get_rte_df(filename):
    df = pd.read_csv(filename, sep='\t',  encoding = 'utf8',keep_default_na=False)
    df=df[['sentence1','sentence2','label']]
    df.columns=['text_a','text_b','label']
    df = df[pd.notnull(df['label'])]  
    df = df[df.label != ""]    
    return df    

def get_rte_data(train_file = DATADIR +'/RTE/train.tsv',
                  dev_file = DATADIR + '/RTE/dev.tsv'):
    
    train = get_rte_df(train_file)
    print("RTE train data size: %d "%(len(train)))        
    dev = get_rte_df(dev_file)
    print("RTE dev data size: %d "%(len(dev)))              
    label_list = np.unique(train['label'].values)

    
    return train,dev,label_list

train,dev,label_list = get_rte_data()

print(label_list)
train.head()
%%time
X_train = train[['text_a','text_b']]
y_train = train['label']

# define model
model = BertClassifier()
model.epochs = 4
model.learning_rate = 3e-5
model.max_seq_length = 96
model.validation_fraction = 0

print('\n',model,'\n')

# fit model
model.fit(X_train, y_train)

# test model on dev
test = dev
X_test = test[['text_a','text_b']]
y_test = test['label']

# make predictions
y_pred = model.predict(X_test)
print("Accuracy: %0.2f%%"%(metrics.accuracy_score(y_pred,y_test) * 100))

target_names = ['entailment', 'not_entailment']
print(classification_report(y_test, y_pred, target_names=target_names))<jupyter_output>Building sklearn classifier...

 BertClassifier(bert_model='bert-base-uncased', epochs=4, eval_batch_size=8,
        fp16=False, gradient_accumulation_steps=1, label_list=None,
        learning_rate=3e-05, local_rank=-1, logfile='bert_sklearn.log',
        loss_scale=0, max_seq_length=96, num_mlp_hiddens=500,
        num_mlp_layers=0, random_state=42, restore_file=None,
        train_batch_size=32, use_cuda=True, validation_fraction=0,
        warmup_proportion=0.1) 

Loading bert-base-uncased model...
Defaulting to linear classifier/regressor
train data size: 2489, validation data size: 0
{'fraction_non_alphanumeric': 0.1293791189732917, 'fraction_numerical': 0.01768990634755463, 'mean_word_length': 4.621832358674464, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '259956', 'license': 'Apache-2.0', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1013, 'n_words': 218, 'path': 'glue_examples/RTE.ipynb', 'repo_name': 'colabnlp/bert-sklearn', 'size': '2883'}