<jupyter_start><jupyter_text>Strategy 1: Using NLTK<jupyter_code>from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import re
lm=WordNetLemmatizer()<jupyter_output><empty_output><jupyter_text>Text Cleaning Method1<jupyter_code>corpus=[]
count=0
for i in range(0,len(df)):
    words1=re.sub('[^a-zA-Z]',' ',(df.iloc[i,:].values)[1])
    words1=words1.lower()
    words1=words1.split()
    word=[lm.lemmatize(word) for word in words1 if word not in set(stopwords.words('english'))]
    words=' '.join(word)
    corpus.append(words)
    count+=1
    print(count)
corpus[0]
len(X_train)<jupyter_output><empty_output><jupyter_text>Method2<jupyter_code>def fun(para):
    dataset=nltk.sent_tokenize(para)
    for i in range(0,len(dataset)):
        dataset[i]=dataset[i].lower()
        dataset[i]=re.sub(r'\W',' ',dataset[i])
        dataset[i]=re.sub(r'\s+',' ',dataset[i])
        words=nltk.word_tokenize(dataset[i])
        words=[lm.lemmatize(word) for word in words if word not in set(stopwords.words('english'))]
        dataset[i]=' '.join(words)
    return " ".join(dataset)
para=[]
count=0
for i in range(0,len(X_train)):
    para.append(fun(X_train[i]))
    count+=1
    print(count)
# for X_test
para_test=[]
count=0
for i in range(0,len(X_test)):
    para_test.append(fun(X_test[i]))
    count+=1
    print(count)
# method 2 trial
para[0]
# first para
X_train[0]
# method 1 trial
corpus[0]<jupyter_output><empty_output><jupyter_text>Bag Of Words Model for corpus method1<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer(max_features=2000)
X=cv.fit_transform(corpus).toarray()
X
y=df.iloc[:,2:].values
y.shape
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.25)
# scaling of model
from sklearn.preprocessing import StandardScaler
sc=StandardScaler()
X_train=sc.fit_transform(X_train)
X_test=sc.transform(X_test)
X_train.shape<jupyter_output><empty_output><jupyter_text>Using KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
classifier=KNeighborsClassifier()
classifier.fit(X_train,y_train)
y_pred=classifier.predict(X_test)
from sklearn.metrics import confusion_matrix
np.sum(np.not_equal(y_test, y_pred))/float(y_test.size)<jupyter_output><empty_output><jupyter_text>Bag of words model for method2<jupyter_code>len(para)
len(para_test)
from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer(max_features=2000)
X_train=cv.fit_transform(para).toarray()
X_test=cv.fit_transform(para_test).toarray()
y_train=df.iloc[0:1973,2:].values
y_train.shape
y_test=df.iloc[1973:,2:].values
y_test.shape
sc=StandardScaler()
X_train=sc.fit_transform(X_train)
X_test=sc.transform(X_test)
X_train.shape<jupyter_output><empty_output><jupyter_text>using KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
knn=KNeighborsClassifier()
knn.fit(X_train,y_train)
y_pred=knn.predict(X_test)
np.sum(np.not_equal(y_test, y_pred))/float(y_test.size)
#Saving the corpus and other models
import pandas as pd
pd.to_pickle(corpus,"./saved_models/corpus")
pd.to_pickle(X_train,"./saved_models/X_train_2")
pd.to_pickle(X_test,"./saved_models/X_test_2")<jupyter_output><empty_output><jupyter_text>Decision Tree Classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import LabelEncoder
lab=LabelEncoder()
y_train
for i in range(5):
    y_train[:,i] = lab.fit_transform(y_train[:,i].astype('str'))
for i in range(5):
    y_test[:,i] = lab.fit_transform(y_test[:,i].astype('str'))
y_train=y_train.astype("float")
y_test=y_test.astype("float")
classifier=DecisionTreeClassifier()
classifier.fit(X_train,y_train)
y_pred=classifier.predict(X_test)
X_train.shape
y_train.shape
np.sum(np.not_equal(y_test, y_pred))/float(y_test.size)
y_pred
y_test<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier
classifier=RandomForestClassifier(n_estimators=50)
classifier.fit(X_train,y_train)
y_pred=classifier.predict(X_test)
np.sum(np.equal(y_test, y_pred))/float(y_test.size)<jupyter_output><empty_output><jupyter_text>XGboost and AdaBoost Classifiers<jupyter_code>from sklearn.ensemble import AdaBoostClassifier
classifier=AdaBoostClassifier()
classifier.fit(X_train,y_train[:,1]
import xgboost as xgb
from sklearn.multiclass import OneVsRestClassifier
from xgboost import XGBClassifier<jupyter_output><empty_output><jupyter_text>Rest vs ALL method to use any classifier for multiple output<jupyter_code>from sklearn.preprocessing import MultiLabelBinarizer
clf = OneVsRestClassifier(XGBClassifier(n_jobs=-1, max_depth=4))
clf.fit(X_train, y_train)
y_pred=clf.predict(X_test)
y_test
y_pred
np.sum(np.equal(y_test, y_pred))/float(y_test.size)<jupyter_output><empty_output><jupyter_text>_________________________________________________________________________________________________________________________ Best model uptil now is xgboost will further do my analysis using ANN and CNN_________________________________________________________________________________________________________________________<jupyter_code>corpus=pd.read_pickle("/home/shashank/Desktop/60udacity/Udadelhi/Personality_detection_from_text_using_nlp/shashank_1/saved_models/corpus")
X_train=pd.read_pickle("/home/shashank/Desktop/60udacity/Udadelhi/Personality_detection_from_text_using_nlp/shashank_1/saved_models/X_train_2")
X_test=pd.read_pickle("/home/shashank/Desktop/60udacity/Udadelhi/Personality_detection_from_text_using_nlp/shashank_1/saved_models/X_test_2")
X_train.shape
y_train=df.iloc[:1973,2:].values
y_test=df.iloc[1973:,2:].values
from keras.models import Sequential
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import ReLU,Dense
from sklearn.preprocessing import LabelEncoder
lab=LabelEncoder()
for i in range(5):
    y_train[:,i] = lab.fit_transform(y_train[:,i].astype('str'))
for i in range(5):
    y_test[:,i] = lab.fit_transform(y_test[:,i].astype('str'))
y_train=y_train.astype("float")
y_test=y_test.astype("float")
model=Sequential()
model.add(Dense(1024,input_dim=2000,activation='relu'))

model.add(Dense(512,activation='relu'))
model.add(Dropout(0.25))

model.add(Dense(256,activation='relu'))
model.add(Dropout(0.25))

model.add(Dense(128,activation='relu'))
model.add(Dropout(0.25))

model.add(Dense(64,activation='relu'))
model.add(Dropout(0.25))

model.add(Dense(32,activation='relu'))
model.add(Dropout(0.25))

model.add(Dense(5,activation='softmax'))

model.compile(loss='categorical_crossentropy',optimizer="adam",metrics=["accuracy"])

model.fit(X_train,y_train,epochs=100,batch_size=64)

loss, accuracy = model.evaluate(X_train, y_train, verbose=False)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy:  {:.4f}".format(accuracy))<jupyter_output>Training Accuracy: 0.3619
Testing Accuracy:  0.1053<jupyter_text>________________________________________ As we can see using ANN doesn't offer pretty impressive results so I will be carry forward my approach with Strategy 2  Word Embeddings<jupyter_code>X=df.iloc[:,1].values
y=df.iloc[:,2:].values
from sklearn.model_selection import train_test_split
sent_train,sent_test,y_train,y_test=train_test_split(X,y,test_size=0.25,shuffle=False)
from keras.preprocessing.text import Tokenizer
tokenizer=Tokenizer(num_words=8000)
tokenizer.fit_on_texts(sent_train)
X_train=tokenizer.texts_to_sequences(sent_train)
X_test=tokenizer.texts_to_sequences(sent_test)
vocab_size=len(tokenizer.word_index)+1
vocab_size
from keras.preprocessing.sequence import pad_sequences

maxlen = 150

X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)
X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)
from keras.models import Sequential
from keras import layers

embedding_dim = 100

model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size, 
                           output_dim=embedding_dim, 
                           input_length=maxlen))
model.add(layers.Flatten())
model.add(layers.Dense(20, activation='relu'))
model.add(layers.Dense(5, activation='softmax'))
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.summary()

model.fit(X_train, y_train,
                    epochs=20,
                    verbose=True,
                    validation_data=(X_test, y_test),
                    batch_size=10)
loss, accuracy = model.evaluate(X_train, y_train, verbose=False)

print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy:  {:.4f}".format(accuracy))<jupyter_output>Training Accuracy: 0.5584
Testing Accuracy:  0.2156<jupyter_text>2 method using globalmaxpool<jupyter_code>from keras.models import Sequential
from keras import layers

embedding_dim = 100

model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size, 
                           output_dim=embedding_dim, 
                           input_length=maxlen))
model.add(layers.GlobalMaxPool1D())
model.add(layers.Dense(20, activation='relu'))
model.add(layers.Dense(5, activation='softmax'))
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.summary()
model.fit(X_train, y_train,
                    epochs=50,
                    verbose=True,
                    validation_data=(X_test, y_test),
                    batch_size=10)
loss, accuracy = model.evaluate(X_train, y_train, verbose=False)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy:  {:.4f}".format(accuracy))
embedding_dim = 200

model = Sequential()
model.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))
model.add(layers.Conv1D(128, 5, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(5, activation='softmax'))
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['categorical_accuracy'])
model.summary()
history = model.fit(X_train, y_train,
                    epochs=40,
                    verbose=True,
                    validation_data=(X_test, y_test),
                    batch_size=32)
loss, accuracy = model.evaluate(X_train, y_train, verbose=False)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy:  {:.4f}".format(accuracy))

maxlen<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1279594656840166, 'fraction_numerical': 0.01953017042837402, 'mean_word_length': 6.476584022038567, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '293759', 'license': 'Apache-2.0', 'n_tokens_mistral': 4054, 'n_tokens_neox': 3853, 'n_words': 566, 'path': 'Personality_detection_from_text_using_nlp/shashank_1/personality.ipynb', 'repo_name': 'bharatnishant/UdacityOpenSource', 'size': '10855'}