<jupyter_start><jupyter_text><jupyter_code># This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 
import time
import numpy as np # linear algebra
from numpy.testing import assert_allclose
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from google.colab import drive
from google.colab import files
import json
import io
import os

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
drive.mount("/content/gdrive/")
print(os.listdir("/content/gdrive/My Drive/Thesis/input"))
root_path='/content/gdrive/My Drive/Thesis/input/'

# Any results you write to the current directory are saved as output.
from keras.models import Model, Sequential, load_model
from keras.layers import Dense, Input, LSTM, GRU, Conv1D, MaxPooling1D, Concatenate, Bidirectional, Flatten, Activation, RepeatVector, Permute, multiply
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger
from keras.utils import np_utils
from keras.initializers import glorot_uniform
from keras.callbacks import Callback
import keras.backend as K
from keras.optimizers import Adam

from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import RandomOverSampler, SMOTE
import matplotlib.pyplot as plt
%matplotlib inline
#initiate default values
maxLen = 128          #maximum length of sentences

master = pd.read_csv('/content/gdrive/My Drive/Thesis/input/master-raw.csv', sep=",")

X = master.copy()
y = pd.DataFrame()
y['aspect'] = master['aspect']
# y['rating'] = master['rating']
import string
tr = str.maketrans(string.punctuation, " "*32)
def modify_phrase(ph, tr):
    ph = ph.lower()
    return ph.translate(tr).strip()
modify_phrase("Hey there! I am here, using whatsapp!!...", tr)
def load_vectors(fname):
    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
    vocab_size, dim = map(int, fin.readline().split())
    word_to_vec_map = {}
    words = set()
    for line in fin:
        tokens = line.rstrip().split(' ')
        words.add(tokens[0])
        word_to_vec_map[tokens[0]] = np.array(tokens[1:], dtype=np.float64)
    i = 1
    words_to_index = {}
    index_to_words = {}
    for w in sorted(words):
        words_to_index[w] = i
        index_to_words[i] = w
        i = i + 1
    return word_to_vec_map, words_to_index, index_to_words, vocab_size, dim
def sentences_to_indices(X, word_to_index, maxLen):
    m = X.shape[0]                                   # number of training examples
    X_indices = np.zeros((m, maxLen))
    
    for i in range(m):
        sentence_words = X[i].lower().strip().split()
        j = 0
        for w in sentence_words:
            if w not in word_to_index:
                w = "person"        #mostly names are not present in vocabulary
            X_indices[i, j] = word_to_index[w]
            if j < maxLen-1:
              j = j + 1
            else:
              break
    
    return X_indices
#load fastText word2vec model
word_to_vec_map, word_to_index, index_to_words, vocab_size, dim= load_vectors('/content/gdrive/My Drive/Thesis/input/wiki.id.vec')
#Set X values into indices 
X_vec = sentences_to_indices(X.sentence, word_to_index, maxLen)
X_pd_vec = pd.DataFrame(X_vec)

#Set Y values into classes


Y_encode = pd.get_dummies(y, columns=['aspect'])
Y_encode['onehotencode'] = Y_encode[Y_encode.columns[:]].apply(
    lambda x: ','.join(x.dropna().astype(int).astype(str)),
    axis=1
)
Y = np.array(Y_encode['onehotencode'])

# labelencoder = LabelEncoder() 
# # label = labelencoder.fit(Y.astype(str))
# Y = labelencoder.fit_transform(np.reshape(Y, (Y.shape[0],1)))

encode = OneHotEncoder(sparse=False)
Y = encode.fit_transform(np.reshape(Y, (Y.shape[0], 1)))
# encode.get_feature_names()
# Y_encode['onehotencode'].value_counts()
# Y[0]
#Over sampling using SMOTE
sm = SMOTE(random_state=42)
X_res, y_res = sm.fit_resample(X_pd_vec, Y)

# ros = RandomOverSampler(random_state=42)
# X_resampled, y_resampled = ros.fit_resample(X_pd_vec, y['aspect'])

#Divide sampling into 75% training and 12.5% testing and 12.5% validation
train_x, test_x, train_y, test_y = train_test_split(X_res, y_res, test_size = 0.4, stratify=y_res, random_state=42)
test_x, val_x, test_y, val_y = train_test_split(test_x, test_y, test_size = 0.5, stratify=test_y, random_state=42)
#test_x, valid_x, test_y, valid_y = train_test_split(test_x, test_y, test_size = 0.2, stratify=test_y, random_state=42)
test_x.shape, test_y.shape
# #Over sampling using SMOTE
# sm = SMOTE(random_state=42)
# train_x_sm, train_y_sm = sm.fit_resample(train_x, Y_train)
# test_x_sm, test_y_sm = sm.fit_resample(test_x, Y_test)

#Translate all values into np.array
# train_x_res = np.array(train_x)
# test_x_res = np.array(test_x)
# valid_x = np.array(val_x)

# train_y_res = np.array(train_y)
# test_y_res = np.array(test_y)
# valid_y = np.array(val_y)

train_x_res = pd.DataFrame(data=train_x)
test_x_res = pd.DataFrame(data=test_x)
valid_x = pd.DataFrame(data=val_x)

train_y_res = pd.DataFrame(data=train_y)
test_y_res = pd.DataFrame(data=test_y)
valid_y = pd.DataFrame(data=val_y)

# Saving the array
train_x_res.to_csv(root_path+'train_x_res.csv', sep=',', encoding='utf-8')
test_x_res.to_csv(root_path+'test_x_res.csv', sep=',', encoding='utf-8')
valid_x.to_csv(root_path+'valid_x.csv', sep=',', encoding='utf-8')

train_y_res.to_csv(root_path+'train_y_res.csv', sep=',', encoding='utf-8')
test_y_res.to_csv(root_path+'test_y_res.csv', sep=',', encoding='utf-8')
valid_y.to_csv(root_path+'valid_y.csv', sep=',', encoding='utf-8')

# np.savetxt(root_path+"train_x_res.csv", train_x_res, delimiter=",")
# np.savetxt(root_path+"test_x_res.csv", test_x_res, delimiter=",")
# np.savetxt(root_path+"valid_x.csv", valid_x, delimiter=",")

# np.savetxt(root_path+"train_y_res.csv", train_y_res, delimiter=",")
# np.savetxt(root_path+"test_y_res.csv", test_y_res, delimiter=",")
# np.savetxt(root_path+"valid_y.csv", valid_y, delimiter=",")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13002659990611798, 'fraction_numerical': 0.009231732123298388, 'mean_word_length': 4.737881508078995, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '605934', 'license': 'MIT', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2331, 'n_words': 521, 'path': 'Initiate_Data.ipynb', 'repo_name': 'yoyochrist/Aspect-Based-Opinion-Mining-Using-Spark', 'size': '6391'}