<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code<jupyter_output><empty_output><jupyter_text>**Scaffolding**  Config<jupyter_code># config.py

# General information
cuda = True
sanity = True
sanitysize = 1100
sanityepoch = 5
load_model = False
# Data information
CONTEXT = 2
input_size = 40 * (2*CONTEXT + 1)
nclasses = 138
# Training parameters
lr = 0.0001
weight_decay = 0.0001
batch_size = 256
max_epoch = 5
patience = 10
# Network parameters
hidden = [330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330]
dropout = 0.02<jupyter_output><empty_output><jupyter_text>Data<jupyter_code># data.py

import paths
import config

import numpy as np
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset



class MyDataset(Dataset):
    def __init__(self, X, Y=None):
        self.Y = Y 
        if self.Y is not None:
          tmp_Y = np.concatenate(tuple(Y[:]), axis=0)
          self.Y = torch.from_numpy(tmp_Y).long()
        self.context = config.CONTEXT
        tmp_X = np.concatenate(tuple(X[:]), axis=0)
        tmp_X = np.pad(tmp_X,((self.context, self.context),(0,0)), mode='reflect')
        self.X = torch.from_numpy(tmp_X).float()
        
    def __len__(self):
      if self.Y is not None:
        return len(self.Y)
      else:
        return self.X.shape[0] - 2*self.context

    def __getitem__(self,index):
        X = self.X[index:index + 2*self.context+1].reshape(-1)  #flatten the input
        if self.Y is not None:
          Y = self.Y[index]
          return X, Y
        else:
          return X


def get_loader(mode="train"):
    if mode == "train":
        data_path = paths.train_data
        labels_path = paths.train_labels
        shuffle = True
    if mode == "val":
        data_path = paths.valid_data
        labels_path = paths.valid_labels
        shuffle = False
    if mode == "test":
        data_path = paths.test_data
        labels_path = None
        shuffle = False
    data = np.load(data_path, allow_pickle=True)
    if config.sanity:
        data = data[:config.sanitysize]

    if labels_path:
        labels = np.load(labels_path, allow_pickle=True)
        if config.sanity:
            labels = labels[:config.sanitysize]

        print(data.shape, labels.shape)
        # dataset = TensorDataset(torch.tensor(data, dtype=torch.float),
        #                         torch.tensor(labels, dtype=torch.long))
        dataset = MyDataset(data, labels)                                
    else:
        dataset = MyDataset(data)
        # dataset = TensorDataset(torch.tensor(data, dtype=torch.float))
    dataloader = DataLoader(dataset, shuffle=shuffle, batch_size=config.batch_size, drop_last=False)

    return dataloader



def get_test_labels():
    labels = np.load(paths.test_labels)
    if config.sanity:
            labels = labels[:config.sanitysize]
    return labels<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># ffnnmodel.py

import config
import paths

import torch
import torch.nn as nn


class FFNN(nn.Sequential):

    def __init__(self):
        l = []
        sizes = [config.input_size] + config.hidden + [config.nclasses]
        for i in range(len(sizes)-2):
            l.append(nn.Linear(sizes[i], sizes[i+1]))
            if config.dropout > 0:
                l.append(nn.Dropout(config.dropout))
            l.append(nn.LeakyReLU())
        l.append(nn.Linear(sizes[-2], sizes[-1]))
        super(FFNN, self).__init__(*l)

    @staticmethod
    def load():
        model = FFNN()
        path = paths.model
        model.load_state_dict(torch.load(path))

        return model

    def save(self):
        path = paths.model
        torch.save(self.state_dict(), path)<jupyter_output><empty_output><jupyter_text>paths.py<jupyter_code>data_prefix = "../float16/"
train_data = data_prefix+"train.npy"
train_labels = data_prefix+"train_labels.npy"
valid_data = data_prefix+"dev.npy"
valid_labels = data_prefix+"dev_labels.npy"
test_data = data_prefix+"test.npy"
test_labels = data_prefix+"test_labels.npy"

model = "../model/model_dict.pt"<jupyter_output><empty_output><jupyter_text>Routine<jupyter_code># coding=utf-8

import config

import torch
import torch.nn as nn
import numpy as np
import time 

device = "cuda" if config.cuda else "cpu"


def run_one_epoch(model, loader, criterion, update=True, optimizer=None):
    cum_loss = 0
    for x, y in loader:
        x = x.to(device)
        y = y.to(device)
        output = model(x)
        loss = criterion(output, y)
        if update:
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
        cum_loss += loss.cpu().detach().numpy().item()
    return cum_loss/len(loader)


def train(model, train_dataloader, val_dataloader):
    max_epoch = config.max_epoch
    if config.sanity:
        max_epoch = config.sanityepoch
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.AdamW(model.parameters(), lr=config.lr)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
    model = model.to(device)
    best_loss = float("+inf")

    print('begin training')
    lr = config.lr
    epoch = 0
    pat = 0
    while True:
        start = time.time()
        epoch += 1
        print("Epoch", epoch)
        model.train()
        train_loss = run_one_epoch(model, train_dataloader, criterion, True, optimizer)
        print("Training loss :", train_loss)
        model.eval()
        val_loss = run_one_epoch(model, val_dataloader, criterion, False)
        print("Validation loss :", val_loss)

        scheduler.step(val_loss)
        if val_loss < best_loss:
            best_loss = val_loss
            model.save()
            pat = 0
        else:
            pat += 1
            if pat >= config.patience:
                print("Early stopping ! IGNORED")
                #break
        if epoch >= max_epoch:
            print('reached maximum number of epochs !')
            break
        stop = time.time()
        print(f"Time :{stop-start}s")


def predict(model, loader):
    l = []
    for x in loader:
        x = x.to(device)
        output = model(x)
        classes = np.argmax(output.cpu().detach().numpy(), axis=1)
        l.append(classes)
    return np.concatenate(l)<jupyter_output><empty_output><jupyter_text>Run<jupyter_code>import data
import ffnnmodel
import routine
import config
import numpy as np

# To run, make sure you have a data/ folder containing MNIST and a model/ folder along with your code/ folder

def run():
    train_loader = data.get_loader("train")
    val_loader = data.get_loader("val")
    test_loader = data.get_loader("test")
    #test_labels = data.get_test_labels()

    if config.load_model:
        model = ffnnmodel.FFNN.load()
    else:
        model = ffnnmodel.FFNN()

    routine.train(model, train_loader, val_loader)
    test_outputs = routine.predict(model, test_loader)
    print((test_outputs))
    try:
    	np.savetxt('prediction.csv', test_outputs, delimiter=",")
    except:
        print("prediction not saved")
    #accuracy = (test_outputs == test_labels).mean()
    #print("Test accuracy :", accuracy)


run()<jupyter_output><empty_output><jupyter_text>**Playing**<jupyter_code>import paths
import numpy as np
import data
import ffnnmodel
import torch

test = data.get_loader("test")
model = ffnnmodel.FFNN.load()


def get_predictions(model, loader):
    final_result = []
    model.eval()
    model.to(torch.device("cuda"))
    for x in loader:
        x = x.to(torch.device("cuda"))
        output = model(x)
        result = np.argmax(output.cpu().detach().numpy(), axis=1)
        final_result.append(result)
    return np.concatenate(final_result)

results = get_predictions(model, test)
tmp = list(zip(range(len(results)), results))
import csv
with open('150x4val206.csv', 'w') as f:
    writer = csv.writer(f)
    writer.writerow(['id','label'])
    writer.writerows(tmp)
import data
import ffnnmodel
import routine
import config
import numpy as np

# To run, make sure you have a data/ folder containing MNIST and a model/ folder along with your code/ folder

def run():
    train_loader = data.get_loader("train")
    val_loader = data.get_loader("val")
    test_loader = data.get_loader("test")
    #test_labels = data.get_test_labels()

    if config.load_model:
        model = ffnnmodel.FFNN.load()
    else:
        model = ffnnmodel.FFNN()

    routine.train(model, train_loader, val_loader)
    test_outputs = routine.predict(model, test_loader)
    print((test_outputs))
    try:
    	np.savetxt('prediction.csv', test_outputs, delimiter=",")
    except:
        print("prediction not saved")
    #accuracy = (test_outputs == test_labels).mean()
    #print("Test accuracy :", accuracy)

if __name__ == "__main__":
    run()<jupyter_output><empty_output><jupyter_text>**From Scratch**  **Setup Google Drive**<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
/content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code<jupyter_text>**Imports**<jupyter_code>%matplotlib inline
import time
import torch
import torch.nn as nn
import numpy as np
import csv
import math
import matplotlib
import matplotlib.pyplot as plt

from tqdm import tqdm
from pathlib import Path
from torch.utils.data import TensorDataset, DataLoader, Dataset<jupyter_output><empty_output><jupyter_text>**Data Loading Cells**  WSJ Dataset<jupyter_code>class WSJDataset(Dataset):
    """
    Expects data in the form of an array of recordings
    each recording has N utterances or elements
    each element is a 10ms frame represented by a 40 sized frequency/energy vector.
    each element can then be interpretted as a phoneme.
    """
    def __init__(self, X, Y=None):
        self.Y = Y 
        if self.Y is not None:
          tmp_Y = np.concatenate(tuple(Y[:]), axis=0)
          self.Y = torch.from_numpy(tmp_Y).long()
        self.context = CONTEXT
        tmp_X = np.concatenate(tuple(X[:]), axis=0)
        tmp_X = np.pad(tmp_X,((self.context, self.context),(0,0)), mode='reflect')
        self.X = torch.from_numpy(tmp_X).float()
        
    def __len__(self):
      if self.Y is not None:
        return len(self.Y)
      else:
        return self.X.shape[0] - 2*self.context

    def __getitem__(self,index):
        X = self.X[index:index + 2*self.context+1].reshape(-1)  #flatten the input
        if self.Y is not None:
          Y = self.Y[index]
          return X, Y
        else:
          return X
def get_loader(mode="train"):
    if mode == "train":
        data_path = TRAIN_DPATH
        labels_path = TRAIN_LPATH
        shuffle = True
    if mode == "val":
        data_path = VALID_DPATH
        labels_path = VALID_LPATH
        shuffle = False
    if mode == "test":
        data_path = TEST_DPATH
        labels_path = None
        shuffle = False
    data = np.load(data_path, allow_pickle=True)

    if SANITY:
        data = data[:SANITYSIZE]

    if labels_path:
        labels = np.load(labels_path, allow_pickle=True)
        if SANITY:
            labels = labels[:SANITYSIZE]
        dataset = WSJDataset(data, labels)                                
    else:
        dataset = WSJDataset(data)
    print(f"{mode} Dataset Size: {len(dataset)}")
    dataloader = DataLoader(dataset, shuffle=shuffle, batch_size=BATCH_SIZE, drop_last=False)

    return dataloader<jupyter_output><empty_output><jupyter_text>Routine :<jupyter_code>def run_one_epoch(model, loader, criterion, update=True, optimizer=None, scheduler=None):
    cum_loss = 0
    plotloss = []
    lrs = []
    i = 0
    corr_preds = 0
    for x, y in tqdm(loader):
        x = x.to(device)
        y = y.to(device)
        output = model(x)
        loss = criterion(output, y)
        if update:
            lrplot = scheduler.get_lr()
            lrs.append(lrplot)
            if i % 1000 == 0:
                print(f"Learning rate: {lrplot}")
            loss.backward()
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
        plotloss.append(loss.cpu().detach().numpy().item()) 
        # Per batch Accuracy data:
        out = np.argmax(output.cpu().detach().numpy(), axis=1)
        true = y.cpu().detach().numpy()
        if WSJ and i % 1000 == 0:
            print(f"Current Batch Accuracy: {(np.sum(true==out)/len(true))*100}%")
        corr_preds += np.sum(true==out)
        i+=1   
        cum_loss += loss.cpu().detach().numpy().item()
    # Overall per Epoch Accuracy
    if update:
        print(f"Training Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%")
        plt.plot(lrs, plotloss)
        plt.xscale("log")
        plt.show()
    else:
        print(f"Validation Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%")
        plt.plot(plotloss)
        plt.show()
    return cum_loss/len(loader)
# def lr_find(model, loader, optimizer, min_lr=1e-7, max_lr=1.0):
#     cum_loss = 0
#     plotloss = []
#     lrs = []
#     i = 0
#     n = math.log(number, 1.01)
#     corr_preds = 0
#     for x, y in tqdm(loader):
#         x = x.to(device)
#         y = y.to(device)
#         output = model(x)
#         loss = criterion(output, y)
#         lrs.append(lrplot)
#         loss.backward()
#         optimizer.step()
#         optimizer.zero_grad()
#         lr = optimizer.param_groups[0]['lr']
#         optimizer.param_groups[0]['lr'] = lr * 1.1
#         plotloss.append(loss.cpu().detach().numpy().item()) 
#         i+=1   
#         cum_loss += loss.cpu().detach().numpy().item()

#     print(f"Training Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%")
#     plt.plot(lrs, plotloss)
#     plt.xscale("log")
#     plt.show()
#     return cum_loss/len(loader)


def train(model, train_dataloader, val_dataloader):
    max_epoch = MAX_EPOCHS
    lr = LEARNING_RATE
    if SANITY:
        max_epoch = SANITYEPOCH
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=0.0001)
    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr=SCH_RATE,steps_per_epoch=len(train_dataloader), epochs=max_epoch)
    model = model.to(device)
    val_losses = []
    train_losses = []
    best_loss = 10000
    print('Begin training')
    for epoch in range(max_epoch):
        start = time.time()
        print("Epoch: ", epoch+1)
        #print("Learning Rate: ", lr)
        model.train()
        train_loss = run_one_epoch(model, train_dataloader, criterion, True, optimizer, scheduler)
        train_losses.append(train_loss)
        print("Training loss: ", train_loss)
        model.eval()
        val_loss = run_one_epoch(model, val_dataloader, criterion, False)
        val_losses.append(val_loss)
        print("Validation loss: ", val_loss)
        stop = time.time()
        if val_loss < best_loss:
            best_loss = val_loss
            print("Saving Model....")
            model.save()
        
        print(f"Time :{stop-start}s")
    return val_losses, train_losses
    


def get_predictions(model, loader):
    final_result = []
    model.eval()
    model.to(torch.device("cuda"))
    for x in loader:
        x = x.to(torch.device("cuda"))
        output = model(x)
        result = np.argmax(output.cpu().detach().numpy(), axis=1)
        final_result.append(result)
    return np.concatenate(final_result)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class FFNN(nn.Sequential):

    def __init__(self):
        l = []
        sizes = [INPUT_SIZE] + HIDDEN_SIZE + [NUM_CLASSES]
        for i in range(len(sizes)-2):
            l.append(nn.Linear(sizes[i], sizes[i+1]))
            l.append(nn.BatchNorm1d(sizes[i+1]))
            l.append(nn.ReLU())
        l.append(nn.Linear(sizes[-2], sizes[-1]))
        super(FFNN, self).__init__(*l)

    @staticmethod
    def load():
        model = FFNN()
        path = MODEL_PATH
        model.load_state_dict(torch.load(path))

        return model

    def save(self):
        path = MODEL_PATH
        torch.save(self.state_dict(), path)<jupyter_output><empty_output><jupyter_text>**Config Cell**(𝐿𝑅 ∗ 𝑊𝐷)/(𝑇𝐵𝑆 ∗ (1 − α)) ≈ 𝑐𝑜𝑛𝑠𝑡𝑎𝑛𝑡α - momentum<jupyter_code># Data Parameters
data_folder = Path("../float16/")
mnist_data_folder = Path("../data/")
WSJ = True
if WSJ:
  TRAIN_DPATH = data_folder/'train.npy' 
  TRAIN_LPATH = data_folder/'train_labels.npy'
  VALID_DPATH = data_folder/'dev.npy'
  VALID_LPATH = data_folder/'dev_labels.npy'
  TEST_DPATH = data_folder/'test.npy'
  MODEL_PATH = Path("../model/tqdm.pt")
  # Model Parameters
  torch.manual_seed(0)
  np.random.seed(0)
  LOAD_MODEL = False
  MAX_EPOCHS = 1
  BATCH_SIZE = 1024
  LEARNING_RATE = 1e-5
  SCH_RATE = 5e-3
  MOMENTUM = 0.9
  CONTEXT = 15
  INPUT_SIZE = 40 * (2*CONTEXT + 1)
  HIDDEN_SIZE = [1240]*6
  NUM_CLASSES = 138
else:
  TRAIN_DPATH = mnist_data_folder/'train.npy' 
  TRAIN_LPATH = mnist_data_folder/'train_labels.npy'
  VALID_DPATH = mnist_data_folder/'dev.npy'
  VALID_LPATH = mnist_data_folder/'dev_labels.npy'
  TEST_DPATH = mnist_data_folder/'test.npy'
  MODEL_PATH = Path("../model/MNISTMODEL.pt")
  # Model Parameters
  torch.manual_seed(0)
  np.random.seed(0)
  LOAD_MODEL = False
  MAX_EPOCHS = 10
  BATCH_SIZE = 64
  LEARNING_RATE = 1e-6
  SCH_RATE = 1e-5
  MOMENTUM = 0.9
  INPUT_SIZE = 784
  HIDDEN_SIZE = [512,512,512,512,512,512,512,512]
  NUM_CLASSES = 10

# Sanity / Training
SANITY = False
SANITYSIZE = 1100
SANITYEPOCH = 2
cuda = True
device = "cuda" if cuda else "cpu"<jupyter_output><empty_output><jupyter_text>Runtime<jupyter_code>def run():
    train_loader = get_loader("train")
    val_loader = get_loader("val")

    if LOAD_MODEL:
        model = FFNN.load()
    else:
        model = FFNN()

    val_losses, train_losses = train(model, train_loader, val_loader)
    plt.plot(val_losses)
    plt.plot(train_losses)
    plt.title("Validation Loss");
    plt.xlabel("Iterations");
    plt.ylabel("Loss");
    return (val_losses, train_losses)
v,t=run();
tempo = np.array([2.3753485043843585,2.0615390134580207,1.8913816471894582,1.7839566740122708,
         1.7331338983593565,1.6891762694626145,1.7151012378208566,1.6674280624498021,
         1.6835154702266057,1.6134144763151805,1.5996342856775632,1.5694274478789532,
         1.6674274412971555,1.5510204764026585,1.6405746139360198,1.5249536400491541,
         1.4834835471528949,1.5473498891700397,1.4986431845209816,1.4536923199892045,
         1.542754511851253,1.4373818435452201,1.3993541067296809,1.433429891051668,
         1.3958047336701191,1.4357629985520335,1.5100324656927224,1.399349873625871,
         1.3719498879981764,1.3774904342311802,1.3632555172298895,1.3484137666044813,
         1.3477942674448995,1.3707056564815117,1.3369793112530852,1.3402825184843756,
         1.356912567579385,1.3421990789247282,1.334889861038237,1.3316648706342236,
         1.336125900889888,1.335822554339062,1.3355693424289876,1.337578166163329,
         1.336651961460258,1.3398560769630201,1.3405852402701521,1.3392708445137198,1.3387861782854253])
plt.plot(tempo)<jupyter_output><empty_output><jupyter_text>Hyperparam stuff<jupyter_code># lr_finder
lr_list = [1e-7, 1e-6, 1e-5, 1e-4, 1e-3]
ttrain_loader = get_loader("train")
vval_loader = get_loader("val")
val_losses = []
train_losses = []
FRESH = True
SANITY = True
SANITYSIZE = 1100
SANITYEPOCH = 10
if FRESH:
    mmodel = FFNN()
    mmodel.save()
for lr in lr_list:
    LEARNING_RATE=lr
    val, trains = train(mmodel, ttrain_loader, vval_loader)
    val_losses.extend(val)
    train_losses.extend(trains)
plt.plot(val_losses)
plt.plot(train_losses)
plt.xlabel("Learning rate")
plt.ylabel("Loss")
#plt.xticks(range(0,SANITYEPOCH*len(lr_list),10), map(str, lr_list))<jupyter_output>train Dataset Size: 667411
val Dataset Size: 675836
Begin training
Epoch:  1
Learning Rate:  1e-07
Training loss:  4.924452876934976
Validation loss:  4.924017782644792
Saving Model....
Time :36.21047854423523s
Epoch:  2
Learning Rate:  1e-07<jupyter_text>Prediction<jupyter_code>model = FFNN.load()
test = get_loader("test")
results = get_predictions(model, test)
tmp = list(zip(range(len(results)), results))
with open('predt680v639.csv', 'w') as f:
    writer = csv.writer(f)
    writer.writerow(['id','label'])
    writer.writerows(tmp)<jupyter_output>test Dataset Size: 223592<jupyter_text>Testing MNIST<jupyter_code>mnist_train = np.load("../data/train.npy")
mnist_train_labels = np.load("../data/train_labels.npy")
mnist_dev = np.load("../data/dev.npy")
mnist_dev_labels = np.load("../data/dev_labels.npy")

print(mnist_train.shape,mnist_train_labels.shape, mnist_dev.shape,mnist_dev_labels.shape)

class MNIST_dataset(Dataset):
  def __init__(self, X, Y):
    self.X = torch.from_numpy(X).float()
    self.Y = torch.from_numpy(Y).long()
  
  def __len__(self):
    return len(self.Y)

  def __getitem__(self, index):
    return self.X[index], self.Y[index]

mnist_train_data = MNIST_dataset(mnist_train, mnist_train_labels)
mnist_dev_data = MNIST_dataset(mnist_dev, mnist_dev_labels)
mnist_train_dataloader = DataLoader(mnist_train_data,batch_size=BATCH_SIZE,shuffle=True)
mnist_dev_dataloader = DataLoader(mnist_dev_data,batch_size=BATCH_SIZE,shuffle=False)

model = FFNN()
val_losses, train_losses = train(model, mnist_train_dataloader, mnist_dev_dataloader)
plt.plot(val_losses)
plt.plot(train_losses)
plt.title("Validation Loss");
plt.xlabel("Iterations");
plt.ylabel("Loss");<jupyter_output>(50000, 784) (50000,) (10000, 784) (10000,)
Begin training
Epoch:  1
Learning rate: [3.9999999999999956e-07]
Training Epoch Accuracy: 27.014%
Training loss:  2.209235254760898
Validation Epoch Accuracy: 64.78%
Validation loss:  1.6154187560840776
Saving Model....
Time :3.879451274871826s
Epoch:  2
Learning rate: [2.801856579744565e-06]
Training Epoch Accuracy: 84.382%
Training loss:  0.5975573842539964
Validation Epoch Accuracy: 91.45%
Validation loss:  0.2898275336357439
Saving Model....
Time :3.834383726119995s
Epoch:  3
Learning rate: [7.6037117232856525e-06]
Training Epoch Accuracy: 92.376%
Training loss:  0.26428379548137143
Validation Epoch Accuracy: 93.82%
Validation loss:  0.21017339728345538
Saving Model....
Time :3.778704881668091s
Epoch:  4
Learning rate: [9.999999176566715e-06]
Training Epoch Accuracy: 94.254%
Training loss:  0.19390377388966967
Validation Epoch Accuracy: 95.51%
Validation loss:  0.1572481741903314
Saving Model....
Time :3.807025194168091s
Epoch:  5
Learnin[...]
{'fraction_non_alphanumeric': 0.12350161868929915, 'fraction_numerical': 0.06759121532942515, 'mean_word_length': 3.196621993758032, 'pattern_counts': {'":': 7, '<': 71, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '624305', 'license': 'MIT', 'n_tokens_mistral': 9314, 'n_tokens_neox': 8247, 'n_words': 1549, 'path': 'CMUHW1P2.ipynb', 'repo_name': 'Translucent504/Playground', 'size': '22858'}