<jupyter_start><jupyter_text>1 for toxic 0 for normal<jupyter_code>toxic = data[data["Label"]==1]
normal = data[data["Label"]==0]
len(toxic),len(normal)
print("Percentage of normal comments = {}".format(len(normal)/len(data)*100))
print("Percentage of toxic comments = {}".format(len(toxic)/len(data)*100))
sns.set()
sns.countplot(data["Label"])
plt.show()
data['char_length'] = data['Review'].apply(lambda x: len(str(x)))
sns.set()
data['char_length'].hist()
plt.show()
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(data['Review'], data['Label'], test_size=0.2, random_state=42)
print(X_train.shape, X_test.shape)
print(y_train.shape, y_test.shape)
import pickle

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
vect = TfidfVectorizer(max_features = 10000, stop_words='english')
#vect = TfidfVectorizer(stop_words='english')
print(vect)
pickle.dump(vect, open("./saved_models/vectorizer.pkl", "wb"))
%%time 

X_train_dtm = vect.fit_transform(X_train.apply(lambda x: np.str_(x)))

X_train_dtm
pickle.dump(vect, open("./saved_models/vectorizer.pkl", "wb"))
vect = pickle.load(open("./saved_models/vectorizer.pkl", 'rb'))
X_train_dtm.shape
%%time
X_test_dtm = vect.transform(X_test.apply(lambda x: np.str_(x)))

X_test_dtm
X_test_dtm.shape
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
logreg = LogisticRegression(C=1, max_iter = 2000)
import pickle

 

# train the model using X_train_dtm & y_train
logreg.fit(X_train_dtm, y_train)
filename = "./saved_models/lr_model.pkl"
pickle.dump(logreg, open(filename, 'wb'))
# compute the training accuracy
y_pred_train = logreg.predict(X_train_dtm)
print('Training accuracy is {}'.format(accuracy_score(y_train, y_pred_train)))
# compute the predicted probabilities for X_test_dtm
y_pred_test = logreg.predict(X_test_dtm)
print('Test accuracy is {}'.format(accuracy_score(y_test,y_pred_test)))
print(confusion_matrix(y_test,y_pred_test))<jupyter_output>Training accuracy is 0.9597241715994386
Test accuracy is 0.9542730058853842
[[29304   256]
 [ 1430  5881]]<jupyter_text>Testing trained model<jupyter_code>import pickle
model = pickle.load(open("./saved_models/lr_model.pkl", 'rb'))
vect = pickle.load(open("./saved_models/vectorizer.pkl", 'rb'))
sen = ["<NAME>...<NAME>uck"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
sen = ["Hey girl...you are beautiful"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
sen = ["Hey girl...you are nice but I think you like anal"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
sen = ["When you want to show your parents a video, and Big Black cock porn from last night is still open"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
sen = ["you get what you fuckin deserve"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
sen = ["When you accidently message your teacher 'FRIENDS is my favrite show..You are a fuckin Hoe'"]
sen_trans = vect.transform(sen)
p = model.predict(sen_trans)[0]
validity = ["allowed","toxic"]
print(validity[p])
! pip install -q pandas-ml
%matplotlib inline
from pandas_ml import ConfusionMatrix
confusion_matrix = ConfusionMatrix(y_true, y_pred)
confusion_matrix.plot(normalized=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14638783269961977, 'fraction_numerical': 0.021727322107550243, 'mean_word_length': 7.408675799086758, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 1, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '64605', 'license': 'MIT', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1313, 'n_words': 279, 'path': 'hatespeech/Model_Training.ipynb', 'repo_name': 'Slainteee/Jatayu-ContentModeration', 'size': '3685'}