<jupyter_start><jupyter_text>Converting preprocessed text into vectors using fasttext embedding.<jupyter_code>from google.colab import drive
import numpy as np
import pandas as pd
drive.mount('/content/gdrive')
bengali_news_after_preprocessing = pd.read_pickle('/content/gdrive/My Drive/Projects/Bengali Text Classification/Bengali_Text_after_preprocessing.pkl')
from sklearn.externals import joblib
filename = '/content/gdrive/My Drive/Projects/Bengali Text Classification/fastText_Bangla_content_full.sav'
loaded_model = joblib.load(filename)
import keras.backend as K
import numpy as np
number_of_sample, max_number_of_words, word_vector_size = 40000, 100, 32
temp = bengali_news_after_preprocessing.loc[:number_of_sample-1,:max_number_of_words-1]<jupyter_output><empty_output><jupyter_text>Alternate method to get weird text<jupyter_code>#ord('ী') #2496
#ord('ু') #2497
c = chr(2496)+chr(2497)
c
temp = temp.replace(['ঘস', 'ফগ', 'ঝবঃ', 'ঋন', 'ঊঘ', '\u09e4', 'ওৎ', 'গথ', 'খঢ', 'ঝ’', 'ং', 'ঔ', 'ডড', 'গঘ','ঐব','ওঃ’ং','ুী','খষ','ি'], None)
X = np.zeros((number_of_sample, max_number_of_words, word_vector_size), dtype=K.floatx())
for i in temp.index:
  X[i,:,:] = loaded_model.wv[temp.loc[i,:]]<jupyter_output><empty_output><jupyter_text>preparing labels from csv<jupyter_code>bengali_news = pd.read_pickle('/content/gdrive/My Drive/Projects/Bengali Text Classification/40k_bangla_newspaper_article.p')
bengali_news_dataframe = pd.DataFrame(bengali_news)
y = bengali_news_dataframe['category']
from sklearn import preprocessing
import keras
import numpy as np
le = preprocessing.LabelEncoder()
le.fit(y)
enc = le.transform(y)
y = keras.utils.to_categorical(enc)<jupyter_output><empty_output><jupyter_text>Train, Validation and Test Split<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(X, y, shuffle = True, test_size=0.125)
X_val, X_test, y_val, y_test = train_test_split(X_val, y_val, shuffle = True, test_size=0.20)<jupyter_output><empty_output><jupyter_text>Training CNN<jupyter_code>from keras.models import Sequential
from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, TensorBoard


model = Sequential()

model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same',
                 input_shape=(max_number_of_words, word_vector_size)))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(MaxPooling1D(pool_size=3))
model.add(Flatten())
model.add(Dense(256, activation='sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(256, activation='sigmoid'))
model.add(Dropout(0.25))
model.add(Dense(256, activation='sigmoid'))
model.add(Dropout(0.25))

model.add(Dense(13, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])
model.summary()
history = model.fit(X_train, y_train, batch_size= 500, shuffle=True, epochs= 100, validation_data=(X_val, y_val))
predicts = model.predict(X_test)
import numpy as np
def decode(le, one_hot):
    dec = np.argmax(one_hot, axis=1)
    return le.inverse_transform(dec)
y_test = decode(le, y_test)
y_preds = decode(le, predicts)
from sklearn import metrics

print(metrics.accuracy_score(y_test, y_preds))

print(metrics.confusion_matrix(y_test, y_preds))

print(metrics.classification_report(y_test, y_preds))
accuracy, val_accuracy = np.array(history.history["acc"]), np.array(history.history["val_acc"])
accuracy, val_accuracy = accuracy.reshape(100,1), val_accuracy.reshape(100,1)
accuracies = np.concatenate((accuracy,val_accuracy),axis=1)
np.savetxt('/content/gdrive/My Drive/Projects/Bengali Text Classification/temp.csv',accuracies,delimiter=",")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14120667522464697, 'fraction_numerical': 0.0251604621309371, 'mean_word_length': 8.45631067961165, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '637459', 'license': 'MIT', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1471, 'n_words': 254, 'path': 'FastText + CNN.ipynb', 'repo_name': 'Shauqi/A-Determination-of-Effective-Word-Embedding-of-Bengali-Language-using-text-classification', 'size': '3895'}