<jupyter_start><jupyter_text><jupyter_code>!wget --header="Host: doc-0g-4s-docs.googleusercontent.com" --header="User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" --header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" --header="Accept-Language: en-US,en;q=0.9" --header="Referer: https://drive.google.com/" --header="Cookie: AUTH_kahsjga3le6prgc14cd9quo9442oc1jl_nonce=1chdsjk9oktgu; _ga=GA1.2.1326684588.1554576252" --header="Connection: keep-alive" "https://doc-0g-4s-docs.googleusercontent.com/docs/securesc/6e55a1dr750nd4vfil7uefcrohjkedpb/ru7p4f5akt1jgo5s4mf6vb0mnecbq8o4/1561305600000/17276416090818950573/07423407162889017215/1JBnuP1GTXvhiTb80OEzN13uV7wMLkktg?e=download&nonce=1chdsjk9oktgu&user=07423407162889017215&hash=moaa7m54vl1edep2idr8c1824mfm2vo1" -O "binary_classifier_data.zip" -c
!unzip binary_classifier_data.zip -d data
import pandas as pd 
import sklearn
df = pd.read_csv("data/training.csv",sep = ";")
df = df.drop(columns = "variable18")
df
# The data was separated some with ; and some with , couldn't get any description of the data so had hard time knowing how to deal with it
# I do a lot of Natural language processing but not a lot with tabular data so couldn't exactly deal with this task


# Check the conversational deep learning model chatbot.
# https://github.com/moustafa-7/ChatBot-Project

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

x = df.iloc[:,:-1].values
y = df.iloc[:,17].values


from sklearn.preprocessing import Imputer

imputer = Imputer(missing_values = 'NaN', strategy = 'mean', axis =0)
x[:,:-1] = imputer.fit_transform(x[:,:-1])

from sklearn.preprocessing import LabelEncoder
label_encoder_x = LabelEncoder()

X[:,0] = label_encoder_x.fit_transform(X[:,0])
X[:,3] = label_encoder_x.fit_transform(X[:,3])
X[:,4] = label_encoder_x.fit_transform(X[:,4])
X[:,5] = label_encoder_x.fit_transform(X[:,5])
X[:,6] = label_encoder_x.fit_transform(X[:,6])
X[:,8] = label_encoder_x.fit_transform(X[:,8])
X[:,9] = label_encoder_x.fit_transform(X[:,9])
X[:,11] = label_encoder_x.fit_transform(X[:,11])
X[:,12] = label_encoder_x.fit_transform(X[:,12])
X[:,12] = label_encoder_x.fit_transform(X[:,12])
X[:,17] = label_encoder_x.fit_transform(X[:,17])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15550935550935552, 'fraction_numerical': 0.09397089397089398, 'mean_word_length': 6.862745098039215, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '155310', 'license': 'MIT', 'n_tokens_mistral': 1054, 'n_tokens_neox': 974, 'n_words': 160, 'path': 'binary_classifier_data.ipynb', 'repo_name': 'moustafa-7/ChatBot-Project', 'size': '2405'}