<jupyter_start><jupyter_text>Personality prediction model This notebook is used to train a model to predict the personality of a user. For see the results of each line, see the archive network on the folder draft.  Data management<jupyter_code># Importing libraries 

import pandas as pd 
import numpy as np
import tensorflow as tf
from pathlib import Path
import os
tf.__version__
place = str (Path(os.getcwd())) + '\\data\\data.csv' # finding database
data = pd.read_csv(place, sep='\t')                  # opening database
df = data.copy()                                    # copying database
df.drop(['dateload'], axis=1, inplace=True)       # dropping dateload column
filt = ~( (df['IE'] == 0 ) | (df['IE'] == 3) | (df['gender'] == 0) )        # Filter for remove invalid rows in column IE and gender
df = df[filt]                           # Applying filter 
# Optimizing DataFrame

intcolumns = df.columns.drop('country')
df[intcolumns] = df[intcolumns].astype(np.int8)
df['country'] = df['country'].astype('category')<jupyter_output><empty_output><jupyter_text>Data preprocessing<jupyter_code># Label Encoding the "country" and "IE" column

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df['country'] = le.fit_transform(df['country'].values)
df['IE'] = le.fit_transform(df['IE'].values)

# Cutting data in parameters and target

X = df.iloc[:, 0:280].values        # all columns except the last one
y = df.iloc[:,280].values           # getting the last column(target)
# Splitting the data into training and test sets

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) 
# Feature Scaling

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
print("X_train >>\n",X_train,
      "\nX_test >>\n",X_test)<jupyter_output>X_train >>
 [[-1.90397584  0.83617006  0.01460497 ... -1.19829693  1.47572684
  -0.51535851]
 [ 0.81699383 -1.60304267 -0.10802687 ...  0.66796069 -0.66739897
  -0.76694216]
 [ 0.81699383  1.21729705  1.41805823 ... -1.19829693 -0.66739897
   0.23939247]
 ...
 [-1.22373342 -1.06946489 -1.63411196 ...  0.66796069  1.47572684
   0.32325368]
 [-0.54349101  0.72183196 -0.74843758 ...  0.66796069  1.47572684
  -0.85080338]
 [-1.22373342  0.34070497  0.91390512 ...  0.66796069 -0.66739897
  -0.76694216]] 
X_test >>
 [[-0.54349101 -0.11664741  1.49981279 ...  0.66796069 -0.66739897
  -0.51535851]
 [ 0.81699383 -0.15476011 -0.72118606 ...  0.66796069 -0.66739897
  -0.68308094]
 [ 0.81699383  1.44597324  0.6141384  ...  0.66796069 -0.66739897
   0.74255978]
 ...
 [ 0.13675141 -0.26909821 -0.42141934 ... -1.19829693 -0.66739897
  -0.51535851]
 [ 0.81699383  0.41693037 -0.20340719 ... -1.19829693 -0.66739897
  -0.51535851]
 [ 0.81699383  1.25540975  1.45893551 ... -1.19829693 -0.66739897
   3.258[...]<jupyter_text>Neural network<jupyter_code>ann = tf.keras.models.Sequential()   # Initializing the ANN
ann.add(tf.keras.layers.Dense(units=140, activation='relu')) # Adding the first hidden layer
ann.add(tf.keras.layers.Dense(units=70, activation='relu')) # Adding the second hidden layer
ann.add(tf.keras.layers.Dense(units=35, activation='relu')) # adding the third hidden layer
ann.add(tf.keras.layers.Dense(units=5, activation='relu')) # Adding the fourth hidden layer
ann.add(tf.keras.layers.Dense(units=1, activation='sigmoid')) # Adding the output layer
ann.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # compiling the ANN
ann.fit(X_train, y_train, batch_size = 32, epochs = 75) # Fitting the ANN to the training set<jupyter_output>Epoch 1/75
134/134 [==============================] - 1s 2ms/step - loss: 0.3104 - accuracy: 0.8781
Epoch 2/75
134/134 [==============================] - 0s 2ms/step - loss: 0.1347 - accuracy: 0.9521
Epoch 3/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0883 - accuracy: 0.9688
Epoch 4/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0492 - accuracy: 0.9850
Epoch 5/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0248 - accuracy: 0.9934
Epoch 6/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0108 - accuracy: 0.9977
Epoch 7/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0041 - accuracy: 0.9995
Epoch 8/75
134/134 [==============================] - 0s 2ms/step - loss: 0.0018 - accuracy: 0.9995
Epoch 9/75
134/134 [==============================] - 0s 2ms/step - loss: 8.2730e-04 - accuracy: 0.9998
Epoch 10/75
134/134 [==============================] - 0s 2ms/step - loss: 5.3469e-04 - accuracy[...]<jupyter_text>Prediction test<jupyter_code>y_pred = ann.predict(X_test)
y_pred
y_pred  = (y_pred > 0.5)
y_pred
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
print(cm)   # ACCURACY IN ~= 93.6%
total = sum(sum(cm))
total
erro = cm[0,1] + cm[1,0]
erro
accuracy = ((erro*(-100))/total)+100
accuracy<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19794750345372014, 'fraction_numerical': 0.17663311624235248, 'mean_word_length': 3.794701986754967, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '382806', 'license': 'MIT', 'n_tokens_mistral': 2424, 'n_tokens_neox': 1880, 'n_words': 430, 'path': 'network.ipynb', 'repo_name': 'Peedrooo/Personality_Prediction_Model', 'size': '5067'}