<jupyter_start><jupyter_text>Save and read embeddings len<jupyter_code>import pickle
with open('len_file', 'wb') as fp:
    pickle.dump(sen_len, fp)

with open ('len_file', 'rb') as fp:
    sen_len = pickle.load(fp)<jupyter_output><empty_output><jupyter_text>`max(sen_len)`: 62  Read labels<jupyter_code>import pandas as pd
import numpy as np
df_train = pd.read_csv('/u/shawnlyu/projects/linguistics/workdir/cleaned_data/se100_newlineremoved_text',index_col=0)
labels = list(df_train.target.values)<jupyter_output><empty_output><jupyter_text>`len(labels)`: 1, 044, 839  Read embeddings  Load embeddings<jupyter_code>import h5py
DATA_ROOT = '/u/shawnlyu/projects/linguistics/workdir/embeddings/elmo_layers.train.hdf5'
with h5py.File(DATA_ROOT,'r') as hf:
    dataset_names = list(hf.keys())<jupyter_output><empty_output><jupyter_text>`len(dataset_names)`: 1, 044, 840  How to access embeddings `h5py_file.get("0")` 0 - 1, 044, 838 for embeddings of each sentences    `h5py_file.get("sentence_to_index")` for sentence and index relations (str)<jupyter_code>h5py_file = h5py.File(DATA_ROOT, 'r')
sen_len = []
for i in range(1044839):
    if i % 10000 == 0:
        print(i)
    sen_len.append(h5py_file.get(str(i)).shape[0])<jupyter_output><empty_output><jupyter_text>Implement CNN-LSTM  Configs<jupyter_code>class Config(dict):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        for k, v in kwargs.items():
            setattr(self, k, v)
    
    def set(self, key, val):
        self[key] = val
        setattr(self, key, val)

config = Config(
    # information about dataset
    training_size=1044839,
    word_embeddings_dim=512,
    sen_len=65, # this should be larger than max-sen-len declared at the top
    
    # pytorch settings    
    seed=1,
    
    # neural nets params
    dataLoader={
    'batch_size': 128,
    'shuffle': True,
    'num_workers': 20,
    },
    lr=3e-4,
    epochs=10,
    hidden_sz=64,
)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.manual_seed(config.seed)<jupyter_output><empty_output><jupyter_text>Implement dataloader for sequential loading<jupyter_code>class Dataset(torch.utils.data.Dataset):
    def __init__(self, list_IDs, labels):
        'Initialization'
        self.labels = labels
        self.list_IDs = list_IDs

    def __len__(self):
        'Denotes the total number of samples'
        return len(self.list_IDs)
    
    def __getitem__(self, index):
        'Generates one sample of data'
        # Select sample
        ID = self.list_IDs[index]

        # Load data and get label
        X = h5py_file.get(ID)
        y = self.labels[ID]

        return X, y
from torch.utils import data
# Generators
IDs = [str(i) for i in range(config.training_size)]
training_set = Dataset(IDs, labels)
training_generator = data.DataLoader(training_set, **config.params)

for epoch in range(config.epochs):
    # Training
    for local_batch, local_labels in training_generator:
        # Transfer to GPU
        local_batch, local_labels = local_batch.to(device), local_labels.to(device)
import torch
import torch.nn.functional as F     # functions<jupyter_output><empty_output><jupyter_text>Set random seed manually to replicate results<jupyter_code>class CNN_LSTM(torch.nn.Module):
    def __init__(self,input_size,hidden_dim):
        super(CNN_LSTM,self).__init__()
        self.input_dim = input_size
        self.hidden_dim = hidden_dim
        self.conv1 = nn.Conv2d(1, 10, kernel_size=(config.sen_len,config.word_embeddings_dim))
        self.conv2 = nn.Conv2d(10, 20, kernel_size=(config.sen_len,config.word_embeddings_dim))
        self.lstm = nn.LSTM(input_size,hidden_dim)
        self.hidden2out = nn.Linear(hidden_dim,1)
        self.hidden = self.init_hidden()
 
    def init_hidden(self):
        return (Variable(torch.zeros(1, 1, self.hidden_dim)),
                Variable(torch.zeros(1, 1, self.hidden_dim)))
 
    def forward(self,seq):
        lstm_out, self.hidden = self.lstm(
            seq.view(len(seq), 1, -1), self.hidden)
        outdat = self.hidden2out(lstm_out.view(len(seq),-1))


net = Net(n_feature=1, n_hidden=10, n_output=1)

print(net)
"""
Net (
  (hidden): Linear (1 -> 10)
  (predict): Linear (10 -> 1)
)
"""<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12260625875758992, 'fraction_numerical': 0.02498832321345166, 'mean_word_length': 3.758888888888889, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '25343', 'license': 'MIT', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1522, 'n_words': 305, 'path': 'analysis/shawn/CNN-LSTM.ipynb', 'repo_name': 'ShawnLYU/Quora-Insincere-Questions-Classification', 'size': '4282'}