<jupyter_start><jupyter_text>Read clean data<jupyter_code>import pandas as pd
positive = pd.read_csv("data/clean/positive.csv", encoding= "utf-8")
positive = list(positive["text"])
slightly_positive = pd.read_csv("data/clean/slightly_positive.csv", encoding= "utf-8")
slightly_positive = list(slightly_positive["text"])
slightly_negative = pd.read_csv("data/clean/slightly_negative.csv", encoding= "utf-8")
slightly_negative = list(slightly_negative["text"])
negative = pd.read_csv("data/clean/negative.csv", encoding= "utf-8")
negative = list(negative["text"])<jupyter_output><empty_output><jupyter_text>Use DeepMoji to encode texts into emotional feature vectors<jupyter_code>from __future__ import print_function, division
import example_helper
import json
import csv
import numpy as np
from deepmoji.sentence_tokenizer import SentenceTokenizer
from deepmoji.model_def import deepmoji_feature_encoding
from deepmoji.global_variables import PRETRAINED_PATH, VOCAB_PATH
maxlen = 30
batch_size = 32

print('Tokenizing using dictionary from {}'.format(VOCAB_PATH))
with open(VOCAB_PATH, 'r') as f:
    vocabulary = json.load(f)
st = SentenceTokenizer(vocabulary, maxlen)
print('Loading model from {}.'.format(PRETRAINED_PATH))
model = deepmoji_feature_encoding(maxlen, PRETRAINED_PATH)
model.summary()
print('Encoding texts..')
tokenized_positive, _, _ = st.tokenize_sentences(positive)
encoding_positive = model.predict(tokenized_positive)
print("Positive shape {}".format(encoding_positive.shape))
print("Positive done")

tokenized_slightly_positive, _, _ = st.tokenize_sentences(slightly_positive)
encoding_slightly_positive = model.predict(tokenized_slightly_positive)
print("Slightly positive shape {}".format(encoding_slightly_positive.shape))
print("Slightly positive done")

tokenized_slightly_negative, _, _ = st.tokenize_sentences(slightly_negative)
encoding_slightly_negative = model.predict(tokenized_slightly_negative)
print("Slightly negative shape {}".format(encoding_slightly_negative.shape))
print("Slightly negative done")

tokenized_negative, _, _ = st.tokenize_sentences(negative)
encoding_negative = model.predict(tokenized_negative)
print("Negative shape {}".format(encoding_negative.shape))
print("Negative done")<jupyter_output>Encoding texts..
Positive shape (31549, 2304)
Positive done
Slightly positive shape (34806, 2304)
Slightly positive done
Slightly negative shape (28486, 2304)
Slightly negative done
Negative shape (37283, 2304)
Negative done<jupyter_text>Train, Validation and Test Sets Split<jupyter_code>from sklearn.model_selection import train_test_split
train_size = 0.8
valid_size = 0.1
test_size = 0.1
random_state = 888
total_labels = 4
def train_valid_test_split(input_matrix, label, train_size, valid_size, test_size, 
                           random_state, total_labels):
    n = input_matrix.shape[0]
    X_train_idx, X_test_idx = train_test_split(list(range(n)), 
                                               test_size = valid_size + test_size, 
                                               random_state = random_state)
    X_valid_idx, X_test_idx = train_test_split(X_test_idx, 
                                               test_size = test_size /(valid_size + test_size), 
                                               random_state = random_state)
    # training set
    X_train = input_matrix[X_train_idx]
    Y_train = np.eye(total_labels, dtype=int)[np.repeat(label, len(X_train_idx))]
    
    # validation set
    X_valid = input_matrix[X_valid_idx]
    Y_valid = np.eye(total_labels, dtype=int)[np.repeat(label, len(X_valid_idx))]
    
    # test set
    X_test = input_matrix[X_test_idx]
    Y_test = np.eye(total_labels, dtype=int)[np.repeat(label, len(X_test_idx))]

    return {"X_train": X_train, 
            "Y_train": Y_train, 
            "X_valid": X_valid, 
            "Y_valid": Y_valid, 
            "X_test": X_test, 
            "Y_test": Y_test,
            "train_idx": X_train_idx,
            "valid_idx": X_valid_idx,
            "test_idx": X_test_idx}

splitted_data_0 = train_valid_test_split(encoding_positive, 
                                         0, train_size, valid_size, test_size, 
                                         random_state, total_labels)
splitted_data_1 = train_valid_test_split(encoding_slightly_positive, 
                                         1, train_size, valid_size, test_size, 
                                         random_state, total_labels)
splitted_data_2 = train_valid_test_split(encoding_slightly_negative, 
                                         2, train_size, valid_size, test_size, 
                                         random_state, total_labels)
splitted_data_3 = train_valid_test_split(encoding_negative, 
                                         3, train_size, valid_size, test_size, 
                                         random_state, total_labels)<jupyter_output><empty_output><jupyter_text>Save train data set<jupyter_code>train_data = np.concatenate((splitted_data_0.get("X_train"),
                             splitted_data_1.get("X_train"),
                             splitted_data_2.get("X_train"),
                             splitted_data_3.get("X_train")), axis=0)
train_label = np.concatenate((splitted_data_0.get("Y_train"),
                              splitted_data_1.get("Y_train"),
                              splitted_data_2.get("Y_train"),
                              splitted_data_3.get("Y_train")), axis=0)
train_index = np.concatenate((np.array(list(map(lambda x: [x,0], splitted_data_0.get("train_idx")))),
                              np.array(list(map(lambda x: [x,1], splitted_data_1.get("train_idx")))),
                              np.array(list(map(lambda x: [x,2], splitted_data_2.get("train_idx")))),
                              np.array(list(map(lambda x: [x,3], splitted_data_3.get("train_idx"))))), axis=0)
np.save("data/train_data.npy", train_data)
np.save("data/train_label.npy", train_label)
np.save("data/train_index.npy", train_index)<jupyter_output><empty_output><jupyter_text>Save valid data set<jupyter_code>valid_data = np.concatenate((splitted_data_0.get("X_valid"),
                             splitted_data_1.get("X_valid"),
                             splitted_data_2.get("X_valid"),
                             splitted_data_3.get("X_valid")), axis=0)
valid_label = np.concatenate((splitted_data_0.get("Y_valid"),
                              splitted_data_1.get("Y_valid"),
                              splitted_data_2.get("Y_valid"),
                              splitted_data_3.get("Y_valid")), axis=0)
valid_index = np.concatenate((np.array(list(map(lambda x: [x,0], splitted_data_0.get("valid_idx")))),
                              np.array(list(map(lambda x: [x,1], splitted_data_1.get("valid_idx")))),
                              np.array(list(map(lambda x: [x,2], splitted_data_2.get("valid_idx")))),
                              np.array(list(map(lambda x: [x,3], splitted_data_3.get("valid_idx"))))), axis=0)
np.save("data/valid_data.npy", valid_data)
np.save("data/valid_label.npy", valid_label)
np.save("data/valid_index.npy", valid_index)<jupyter_output><empty_output><jupyter_text>Save test data set<jupyter_code>test_data = np.concatenate((splitted_data_0.get("X_test"),
                             splitted_data_1.get("X_test"),
                             splitted_data_2.get("X_test"),
                             splitted_data_3.get("X_test")), axis=0)
test_label = np.concatenate((splitted_data_0.get("Y_test"),
                              splitted_data_1.get("Y_test"),
                              splitted_data_2.get("Y_test"),
                              splitted_data_3.get("Y_test")), axis=0)
test_index = np.concatenate((np.array(list(map(lambda x: [x,0], splitted_data_0.get("train_idx")))),
                             np.array(list(map(lambda x: [x,1], splitted_data_1.get("train_idx")))),
                             np.array(list(map(lambda x: [x,2], splitted_data_2.get("train_idx")))),
                             np.array(list(map(lambda x: [x,3], splitted_data_3.get("train_idx"))))), axis=0)
np.save("data/test_data.npy", test_data)
np.save("data/test_label.npy", test_label)
np.save("data/test_index.npy", test_index)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1160692807312966, 'fraction_numerical': 0.014433485686793361, 'mean_word_length': 2.8982653539615564, 'pattern_counts': {'":': 9, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '202403', 'license': 'MIT', 'n_tokens_mistral': 2723, 'n_tokens_neox': 2592, 'n_words': 397, 'path': 'emotion detection/Train Valid Test Split.ipynb', 'repo_name': 'jhxu0416/DeepMoji', 'size': '8314'}