<jupyter_start><jupyter_text>BaselineTesting Task (Round-2) : 3{'score_acc': 0.5167118337850045, 'score_secondary_spearman': 0.3149635036496349, 'meta': {'MRR': 0.895, 'Precision': 0.5167118337850045}}  Models  BioBERT<jupyter_code>import torch
from transformers import BertTokenizer, BertModel

# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows
import logging
#logging.basicConfig(level=logging.INFO)

import matplotlib.pyplot as plt
# % matplotlib inline

# Load pre-trained model tokenizer (vocabulary)
# tokenizer = BertTokenizer.from_pretrained("dmis-lab/biobert-large-cased-v1.1")
# tokenizer = BertTokenizer.from_pretrained("dmis-lab/biobert-v1.1")
tokenizer = BertTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")


# model = BertModel.from_pretrained('dmis-lab/biobert-large-cased-v1.1',
#                                   output_hidden_states = True, # Whether the model returns all hidden-states.
#                                   )
def get_bert_sentence_embedding(sentence):
    marked_text = "[CLS] " + sentence + " [SEP]"
    tokenized_text = tokenizer.tokenize(marked_text)

    # Map the token strings to their vocabulary indeces.
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    segments_ids = [1] * len(tokenized_text)
    tokens_tensor = torch.tensor([indexed_tokens])
    segments_tensors = torch.tensor([segments_ids])
    with torch.no_grad():
        outputs = model(tokens_tensor, segments_tensors)

        # Evaluating the model will return a different number of objects based on 
        # how it's  configured in the `from_pretrained` call earlier. In this case, 
        # becase we set `output_hidden_states = True`, the third item will be the 
        # hidden states from all layers. See the documentation for more details:
        # https://huggingface.co/transformers/model_doc/bert.html#bertmodel
        CLS = outputs[0][0]
        hidden_states = outputs[2]

    token_embeddings = torch.stack(hidden_states, dim=0)
    token_embeddings = torch.squeeze(token_embeddings, dim=1)
    # print(len(hidden_states.shape))
    token_vecs = hidden_states[-2][0]

    # Calculate the average of all n token vectors.
    sentence_embedding = torch.mean(token_vecs, dim=0)


    return sentence_embedding

def get_CLS(sentence):
    marked_text = "[CLS] " + sentence + " [SEP]"
    tokenized_text = tokenizer.tokenize(marked_text)

    # Map the token strings to their vocabulary indeces.
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    segments_ids = [1] * len(tokenized_text)
    tokens_tensor = torch.tensor([indexed_tokens])
    segments_tensors = torch.tensor([segments_ids])
    outputs = model(tokens_tensor, segments_tensors)
    CLS = outputs[0][:,0,:]
    return CLS

def get_full_sentence_embedding(sentence):
    embeddings = []
    e = 0
    max_size = 1024#512
    for i in range(int(len(sentence)/max_size)+1):
#         print(i, max_size*(i+1), len(sentence)/max_size)
#         e = get_bert_sentence_embedding(sentence[i*max_size:max_size*(i+1)])
        e = get_CLS(sentence[i*max_size:max_size*(i+1)])
#         print(e)
        embeddings.append(e)
    embedding = torch.mean(torch.stack(embeddings), dim=0)
    print(embedding)
    return embedding
K = 7
q = get_bert_sentence_embedding(QA[K].question)
ans = [get_full_sentence_embedding(a) for a in QA[K].answers] # 512 is the maximum length
from scipy.spatial.distance import cosine
print('Label,Rank,Similarity')
for i,a in enumerate(ans):
    sim = 1-cosine(q.detach(), a.detach())
    print(QA.labels[K][i], QA.references[K][i], sim)
marked_text = "[CLS] " + QA[K].question + " [SEP]"
tokenized_text = tokenizer.tokenize(marked_text, add_special_tokens=True)
print(tokenized_text)
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
segments_ids = [1] * len(tokenized_text)
tokens_tensor = torch.tensor([indexed_tokens])
segments_tensors = torch.tensor([segments_ids])
a = model(tokens_tensor)
a[0][:,0,:][0]
# train_features, test_features, train_labels, test_labels = train_test_split(features, labels)
# lr_clf = LogisticRegression()
# lr_clf.fit(train_features, train_labels)<jupyter_output><empty_output><jupyter_text>Train and test Datasets<jupyter_code>sentences = [[q.question, a]  for q in QA for a in q.answers]
flatten   = lambda t: [item for sublist in t for item in sublist]
labels    = flatten([q.labels for q in QA])

sentences_test = [[q.question, a]  for q in QA_test for a in q.answers]
labels_test    = flatten([q.labels for q in QA_test]) 
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

BATCH_SIZE = 8
BATCH_SIZE_TEST = 64
max_len_seq = 512

class MEDIQA_Dataset(Dataset):
    def __init__(self, X, y, transform=None):
        self.X = []
        self.y = np.array(y)
        for q, a in X:
            _q = tokenizer.convert_tokens_to_ids(tokenizer.tokenize("[CLS] " + q + " [SEP]"))[:max_len_seq]
            _q += [0]*(max_len_seq-len(_q))
            _a = tokenizer.convert_tokens_to_ids(tokenizer.tokenize("[CLS] " + a + " [SEP]"))[:max_len_seq]
            _a += [0]*(max_len_seq-len(_a))
            self.X.append([_q, _a])
        self.X = np.array(self.X)
        
    def __len__(self):
        return self.X.shape[0]
    
    def __getitem__(self, index):
        score = torch.FloatTensor([self.y[index]])
    
        q = torch.LongTensor(self.X[index][0])
        a = torch.LongTensor(self.X[index][1])
        
        return score, q, a

# Create train dataset
train_dataset = MEDIQA_Dataset(X=sentences, y=labels)
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
# Create test dataset
test_dataset = MEDIQA_Dataset(X=sentences_test, y=labels)
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE_TEST, shuffle=False)
len(sentences)
max_len_seq = 0
lenghts = []
for q,a in sentences:
    lenghts.append(len(a))
    if len(a)>max_len_seq:
        max_len_seq = len(a)
max_len_seq
sorted(lenghts, reverse=True)
# len(bert_clf.bert.encoder.layer)
# bert_clf.bert.config.hidden_size

import torch.nn as nn
from torch.nn import functional as F

class MEDIQA_Model(nn.Module):
    def __init__(self):
        super(MEDIQA_Model, self).__init__()
#         self.bert = BertModel.from_pretrained('dmis-lab/biobert-v1.1')
        self.bert = BertModel.from_pretrained('emilyalsentzer/Bio_ClinicalBERT')
        modules = [self.bert.embeddings, *self.bert.encoder.layer[:-1]] #Replace 5 by what you want
        for module in modules:
            for param in module.parameters():
                param.requires_grad = False
        self.linear1 = nn.Linear(2*self.bert.config.hidden_size, 512)
        self.linear2 = nn.Linear(512, 128)
        self.linear3 = nn.Linear(128, 1)
        self.sigmoid = nn.Sigmoid()


    def forward(self, q, a):
        
#         _, pooled_output = self.bert(tokens, output_all=False)
#         print(q.shape, a.shape)
        CLS1 = self.get_CLS(q)
        CLS2 = self.get_CLS(a)
#         print('CLS:', CLS1.shape, CLS2.shape)
        x = torch.cat([CLS1, CLS2], dim=1)
#         print('concat:', x.shape)
        x = F.dropout(x, 0.2)
        x = self.linear1(x)
        x = nn.LeakyReLU(0.1)(x)
        x = F.dropout(x, 0.2)
        x = self.linear2(x)
        x = nn.LeakyReLU(0.1)(x)
        x = F.dropout(x, 0.1)
        x = self.linear3(x)
        prob = self.sigmoid(x)
        return prob, CLS1, CLS2


    def get_CLS(self, indexed_tokens):
        # Map the token strings to their vocabulary indeces.
#         indexed_tokens = tokenizer.convert_tokens_to_ids(tokeniEARLY_STOPPINGtext)
#         segments_ids = [1] * len(indexed_tokens)
        tokens_tensor = indexed_tokens
#         segments_tensors = torch.tensor([segments_ids])
        outputs = self.bert(tokens_tensor)
        CLS = outputs[0][:,0,:]
        return CLS

    def get_full_sentence_embedding(self, sentence):
        embeddings = []
        e = 0
        max_size = 1024#512
        for i in range(int(len(sentence)/max_size)+1):
    #         print(i, max_size*(i+1), len(sentence)/max_size)
    #         e = get_bert_sentence_embedding(sentence[i*max_size:max_size*(i+1)])
            e = self.get_CLS(sentence[i*max_size:max_size*(i+1)])
    #         print(e)
            embeddings.append(e)
        embedding = torch.mean(torch.stack(embeddings), dim=0)
        print(embedding)
        return embedding

cpu = torch.device('cpu')
cuda = torch.device('cuda')
device = cuda if torch.cuda.is_available() else cpu
# del bert_clf
# torch.cuda.empty_cache()
import gc
gc.collect()
print(torch.cuda.memory_allocated())
print(torch.cuda.max_memory_allocated())
bert_clf = MEDIQA_Model()
bert_clf = bert_clf.to(device)
from sklearn.metrics import accuracy_score
def get_test_acc(model, return_probs_and_labels=False):
    model.eval()
    pred_probs = []
    with torch.no_grad():
        for s,q,a in tqdm.tqdm(test_loader):
            logits, _, _ = model(q.to(device),a.to(device))
            pred_probs.extend(logits.to('cpu'))
        pred_probs  = np.array([x.item() for x in pred_probs])
        pred_labels = (pred_probs > 0.5).astype(np.int16)
    acc = accuracy_score(labels_test, pred_labels)
    if return_probs_and_labels:
        return acc, pred_probs, pred_labels
    else:
        return acc
    
a = [1,2,3,4]
a[-3:]
import tqdm
# bert_clf = MEDIQA_Model()
# bert_clf = bert_clf.to(device)
optimizer = torch.optim.Adam(bert_clf.parameters(), lr=1e-4)
bert_clf.train()
EPOCHS = 200
EARLY_STOPPING = 5
loss_func = nn.BCELoss()
train_losses, test_losses, test_acc = [], [], []
for epoch_num in range(EPOCHS):
    losses = []
    for step_num, batch_data in enumerate(train_loader):
        y_true, questions, answers = batch_data #tuple(t.to(device) for t in batch_data)
        if questions.shape != answers.shape: continue
        logits, _, _ = bert_clf(questions.to(device), answers.to(device))
        loss = loss_func(logits, y_true.to(device))
        
        bert_clf.zero_grad()
        loss.backward()
        print('step', loss.item(), end="\r")
        losses.append(loss.item())
        optimizer.step()
        del y_true
        del questions
        del answers
        torch.cuda.empty_cache()
    
    print()
    print(f'Epoch {epoch_num+1} mean loss:', np.mean(losses))
    train_losses.append(np.mean(losses))
    acc, probs_labels, _ = get_test_acc(bert_clf, return_probs_and_labels=True)
    test_acc.append(acc)
    test_loss = loss_func(torch.from_numpy(probs_labels), torch.from_numpy(np.array(labels_test, dtype=np.double))).item()
    test_losses.append(test_loss)
    print(f'Test acc:', acc, '  Test loss:', test_loss)
    print()
    
    if len(test_acc) <= 1 or acc > max(test_acc[:-1]):
        torch.save(bert_clf.state_dict(), 'checkpoints/model')
    if len(test_acc) > EARLY_STOPPING and test_acc[-(EARLY_STOPPING+1)] > max(test_acc[-EARLY_STOPPING:]):
        print('Early stopping')
        # recover best execution
        model = MEDIQA_Model()
        model.load_state_dict(torch.load('checkpoints/model'))
        break
import matplotlib.pyplot as plt

plt.plot(train_losses, label='train loss')
plt.plot(test_losses, label='test loss')
plt.plot(test_acc, label='test acc')
plt.legend()
plt.show()
# Save model
torch.save(bert_clf.state_dict(), 'models/mediqa_model_biobert_2')
# Load model
# model = MEDIQA_Model()
# model.load_state_dict(torch.load(PATH))
# model.eval()
acc, probs, y_pred = get_test_acc(model.to(device), return_probs_and_labels=True)
def get_ranking_predictions(probs, y):
    rankings  = []
    entailed = []
    i_start  = 0
    for i, q in enumerate(QA_test):
        rankings.append(1- np.array(probs[i_start:i_start+len(q.answers)]))
        entailed.append(y[i_start:i_start+len(q.answers)])
        i_start += len(q.answers)
        assert len(rankings[i] == len(QA_test[i].answer_ids))
        assert len(entailed[i] == len(QA_test[i].answers))
    return rankings, entailed
ranking_pred, labels_pred = get_ranking_predictions(probs, y_pred)
QA_test.output_predictions(ranking_pred, labels_pred, file='test_biobert2')
evaluate('test_biobert2')
bert_clf.eval()
pred_labels = []
bert_clf.to(device)
with torch.no_grad():
    for s,q,a in tqdm.tqdm(test_loader):
        logits = bert_clf(q.to(device),a.to(device))
        pred_labels.extend(logits.to('cpu'))
    pred_labels = np.array([x.item() for x in pred_labels])
    pred = (pred_labels > 0.5).astype(np.int16)
pred
from sklearn.metrics import accuracy_score

accuracy_score(labels_test, pred)
torch.cat([torch.tensor([1,2,3]), torch.tensor([1,2,3])], dim=0)
torch.tensor([1,2,3]).shape
for b in train_loader:
    print(b)
from sentence_transformers import SentenceTransformer
from scipy.spatial.distance import cosine

cpu = torch.device('cpu')
cuda = torch.device('cuda')
device = cuda if torch.cuda.is_available() else cpu

model_names = [
    'bert-base-nli-stsb-mean-tokens',
    'bert-large-nli-stsb-mean-tokens',
    'roberta-base-nli-stsb-mean-tokens',
    'roberta-large-nli-stsb-mean-tokens',
    'distilbert-base-nli-stsb-mean-tokens'
]

for name in model_names:
    model = SentenceTransformer(name)
    model = model.to(cuda)

    representations_a = []
    representations_b = []

    with torch.no_grad():
        for (sent_a, sent_b) in tqdm(test_sentences, desc='Embedding Sentences', ncols=800):
            sentences_embeddings = model.encode([sent_a, sent_b])
            representations_a.append(sentences_embeddings[0])
            representations_b.append(sentences_embeddings[1])

    obtained_scores = []
    for idx, (repr_a, repr_b) in enumerate(zip(representations_a, representations_b)):
        score = 1 - cosine(repr_a, repr_b)
        obtained_scores.append(score)

    corr_score = pearsonr(test_scores[:len(obtained_scores)], obtained_scores)[0]
    print(f'{name}: {corr_score}')<jupyter_output>100%|██████████| 405M/405M [00:07<00:00, 51.2MB/s]<jupyter_text>BioELMohttps://docs.allennlp.org/v1.0.0rc5/tutorials/how_to/elmo/https://github.com/allenai/allennlp/blob/main/allennlp/modules/elmo.py<jupyter_code># ! pip install allennlp
import torch
from allennlp.modules.elmo import Elmo, batch_to_ids
# ! pip install allennlp
# ! pip install allennlp-models
cpu = torch.device('cpu')
cuda = torch.device('cuda')
device = cuda if torch.cuda.is_available() else cpu
from allennlp.modules.elmo import Elmo
# elmo = Elmo(
#     options_file='https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json', 
#     weight_file='https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5',
#     num_output_representations=3,
#     dropout=0
    
# )

bioelmo = Elmo(
    options_file='bioelmo/biomed_elmo_options.json', 
    weight_file='bioelmo/biomed_elmo_weights.hdf5',
    num_output_representations=3,
    dropout=0
)
bioelmo = bioelmo.to(device)
sentences = [['First', 'sentence', '.'], ['Another', '.'], ["I", "ate", "a", "carrot", "for", "breakfast"]]
character_ids = batch_to_ids(sentences).to(device)
embeddings = bioelmo(character_ids)
character_ids
def get_elmo_embedding(sentence):
    tokens = nltk.word_tokenize(sentence)
#     print(tokens)
    sentences = [tokens]
    character_ids = batch_to_ids(sentences).to(device)
    return bioelmo(character_ids)['elmo_representations'][2].mean(dim=0).mean(dim=0)
    
embeddings['elmo_representations'][0].mean(dim=0).mean(dim=0)
K = 0
q = get_elmo_embedding(QA[K].question)
ans = [get_elmo_embedding(a) for a in QA[K].answers]
from scipy.spatial.distance import cosine
print('Label,Rank,Similarity')
for i,a in enumerate(ans):
    sim = 1-cosine(q.detach().cpu(), a.detach().cpu())
    print(QA.labels[K][i], QA.references[K][i], sim)
QA[0].labels
sentences = [[q.question, a]  for q in QA for a in q.answers]
flatten   = lambda t: [item for sublist in t for item in sublist]
labels    = flatten([q.labels for q in QA])

sentences_test = [[q.question, a]  for q in QA_test for a in q.answers]
labels_test    = flatten([q.labels for q in QA_test]) 
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

BATCH_SIZE = 8
BATCH_SIZE_TEST = 64
max_len_seq = 512

class MEDIQA_Dataset2(Dataset):
    def __init__(self, X, y, transform=None):
        self.X = []
        self.y = np.array(y)
        for q, a in X:
            _q = batch_to_ids([nltk.word_tokenize(q)])
            _a = batch_to_ids([nltk.word_tokenize(a)])
            self.X.append([_q, _a])
#         self.X = np.array(self.X)
        
    def __len__(self):
#         return self.X.shape[0]
        return len(self.X)
    
    def __getitem__(self, index):
        score = torch.FloatTensor([self.y[index]])
    
        q = torch.LongTensor(self.X[index][0])
        a = torch.LongTensor(self.X[index][1])
        
        return score, q, a

# Create train dataset
train_dataset = MEDIQA_Dataset2(X=sentences, y=labels)
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
# Create test dataset
test_dataset = MEDIQA_Dataset2(X=sentences_test, y=labels)
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE_TEST, shuffle=False)
import torch.nn as nn
from torch.nn import functional as F

class MEDIQA_Model_bioELMo(nn.Module):
    def __init__(self):
        super(MEDIQA_Model_bioELMo, self).__init__()
        self.bioelmo = Elmo(
            options_file='bioelmo/biomed_elmo_options.json', 
            weight_file='bioelmo/biomed_elmo_weights.hdf5',
            num_output_representations=3,
            dropout=0
        )
        for param in self.bioelmo.parameters():
            param.requires_grad = False
#         self.bioelmo = bioelmo.to(device)
#         modules = [self.bert.embeddings, *self.bert.encoder.layer[:-1]] #Replace 5 by what you want
#         for module in modules:
#             for param in module.parameters():
#                 param.requires_grad = False
        self.linear1 = nn.Linear(2*1024, 128)
        self.linear2 = nn.Linear(128, 1)
        self.sigmoid = nn.Sigmoid()


    def forward(self, q, a):
        
#         _, pooled_output = self.bert(tokens, output_all=False)
#         print(q.shape, a.shape)
        q_emb = self.get_elmo_embedding(q)
        a_emb = self.get_elmo_embedding(a)
#         print('CLS:', CLS1.shape, CLS2.shape)
        x = torch.cat([q_emb, a_emb], dim=0)
#         print('concat:', x.shape)
        x = F.dropout(x, 0.2)
        x = self.linear1(x)
        x = nn.LeakyReLU(0.1)(x)
        x = F.dropout(x, 0.1)
        x = self.linear2(x)
        prob = self.sigmoid(x)
        return prob, q_emb, a_emb


    def get_elmo_embedding(self, sentence):
#         tokens = nltk.word_tokenize(sentence)
#     print(tokens)
#         sentences = [tokens]
#         character_ids = batch_to_ids(sentence).to(device)
        return self.bioelmo(sentence)['elmo_representations'][2].mean(dim=0).mean(dim=0)
bioelmo_clf = MEDIQA_Model_bioELMo()
bioelmo_clf = bioelmo_clf.to(device)
from sklearn.metrics import accuracy_score
def get_test_acc(model, return_probs_and_labels=False):
    model.eval()
    pred_probs = []
    with torch.no_grad():
        for s,(q,a) in tqdm.tqdm(zip(test_dataset.y, test_dataset.X)):
            logits, _, _ = model(q.to(device),a.to(device))
            pred_probs.extend(logits.to('cpu'))
        pred_probs  = np.array([x.item() for x in pred_probs])
        pred_labels = (pred_probs > 0.5).astype(np.int16)
    acc = accuracy_score(labels_test, pred_labels)
    if return_probs_and_labels:
        return acc, pred_probs, pred_labels
    else:
        return acc
# Train
def batch(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx + n, l)]

import tqdm
import random
# bert_clf = MEDIQA_Model()
# bert_clf = bert_clf.to(device)
optimizer = torch.optim.Adam(bioelmo_clf.parameters(), lr=1e-3)
bioelmo_clf.train()
EPOCHS = 100
loss_func = nn.BCELoss()
train_losses, test_losses, test_acc = [], [], []
N = len(train_dataset.y)
for epoch_num in range(EPOCHS):
    losses = []
    
    for step_num, batch_data in enumerate(list(zip(train_dataset.y, train_dataset.X))):
        y_true, (questions, answers) = batch_data #tuple(t.to(device) for t in batch_data)
#         q_e = bioelmo_clf.get_elmo_embedding(questions.to(device))
#         a_e = bioelmo_clf.get_elmo_embedding(answers.to(device))
#         print(q_e)
#         print(a_e)
#         x = torch.cat([q_e, a_e], dim=0)
#         print(y_true)
        logits, _, _ = bioelmo_clf(questions.to(device), answers.to(device))
        y_true = torch.from_numpy(np.array([y_true], dtype=np.float32))
        loss = loss_func(logits, y_true.to(device))
        
        bioelmo_clf.zero_grad()
        loss.backward()
        print(f'step {step_num}/{N}', loss.item(), end="\r")
        losses.append(loss.item())
        optimizer.step()
        del y_true
        del questions
        del answers
        torch.cuda.empty_cache()
    print()
    print(f'Epoch {epoch_num+1}:', np.mean(losses))
    train_losses.append(np.mean(losses))
#     acc = get_test_acc(bioelmo_clf)
    acc, probs_labels, _ = get_test_acc(bioelmo_clf, return_probs_and_labels=True)
    test_acc.append(acc)
    test_loss = loss_func(torch.from_numpy(probs_labels), torch.from_numpy(np.array(labels_test, dtype=np.double))).item()
    test_losses.append(test_loss)
    test_acc.append(acc)
    print(f'Test acc:', acc, '  Test loss:', test_loss)
    print()
    
    if len(test_acc) > 5 and test_acc[-6] > max(test_acc[-5:]):
        print('Early stopping')
        break<jupyter_output>step 1699/1701 0.6242034435272217443.6841151714324951 0.10314517468214035 0.15529237687587738 0.339221715927124 0.3030651807785034 2.9525387287139893 0.07003825902938843 0.12832801043987274 0.043231889605522156 0.18832813203334808 0.21925053000450134 0.6420642733573914 0.7310922741889954 0.18545939028263092 0.1863744854927063 0.16231219470500946 0.4953307509422302 0.28373128175735474 0.9389384984970093 0.6665723919868469 0.5955307483673096 0.15899860858917236 2.0907347202301025 0.7578749060630798 0.32404041290283203 0.4283123314380646 0.012842393480241299 1.1186972856521606 0.896552562713623 0.5974249243736267 0.4825071096420288 1.257737159729004 1.2403526306152344 0.6301290988922119 0.6242996454238892 0.5066736340522766 0.9388933777809143 0.8013446927070618 0.7967394590377808 0.6420504450798035 0.2655837833881378 0.3406597971916199 0.14492468535900116 0.3025912940502167 0.260383665561676 1.2056822776794434 0.41638943552970886 0.6138029098510742 0.08509863913059235 0.4399171769618988 0[...]
{'fraction_non_alphanumeric': 0.12697365527554208, 'fraction_numerical': 0.0559000747679993, 'mean_word_length': 3.5889001009081736, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 20, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '69919', 'license': 'MIT', 'n_tokens_mistral': 9228, 'n_tokens_neox': 8381, 'n_words': 1478, 'path': 'old/.ipynb_checkpoints/Models-a2a-checkpoint.ipynb', 'repo_name': 'gonzalorecio/MAI-HLE', 'size': '22737'}