<jupyter_start><jupyter_text>Adult Dataset[EDA for This dataset](https://github.com/romilsiddhapura/ml_practices_2018/blob/master/Labs/Lab1/Romil/Notebooks/adult_EDA.ipynb)  [Data for the same](https://github.com/romilsiddhapura/ml_practices_2018/blob/master/Labs/Lab1/Romil/Data/adult.csv)<jupyter_code>#Importing some necessary packages
import numpy as np
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from matplotlib.patches import Patch
from sklearn.preprocessing import LabelBinarizer
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn import metrics
from yellowbrick.features import ParallelCoordinates
from yellowbrick import classifier
from yellowbrick.classifier import ClassificationReport, ConfusionMatrix

from sklearn.naive_bayes import GaussianNB
#reading a file and adding column names to it.
names=['age','workclass','fnlwgt','education','education_num','marital_status','occupation','relationship','race','sex','capial_gain','capital_loss','hours_per_week','native_country','salary']
data = pd.read_csv('../../Lab1/Romil/Data/adult.csv',header=None,names=names)
data.head()
#Replacing '?' with NaN value
data = data.replace('?',np.NaN)
#Dropping the null rows
data = data.dropna()
#Checking null values in all column
data.isnull().sum()<jupyter_output><empty_output><jupyter_text>Changing some of the column type into category<jupyter_code>data["workclass"] = data["workclass"].astype('category')
#data.workclass = data.workclass.cat.codes
data.marital_status = data.marital_status.astype('category')#.cat.codes
data.race = data.race.astype('category')#.cat.codes
data.relationship = data.relationship.astype('category')#.cat.codes
data.occupation = data.occupation.astype('category')#.cat.codes
data.native_country = data.native_country.astype('category')#.cat.codes
data.sex = data.sex.astype('category')#.cat.codes
data.salary = data.salary.astype('category').cat.codes

data.head()<jupyter_output><empty_output><jupyter_text>Correlation map<jupyter_code>sns.set(rc={'figure.figsize':(15,8)})
sns.heatmap(data.corr(),annot=True)
data.dtypes
#Taking educatoin and capital gain as a main features
X = data[['education_num','capial_gain']]
y = data.salary
#Standardizing the data
sc = StandardScaler()
sc.fit(X)
X_std = sc.transform(X)

#Splitting dataset into train and test 
X_train, X_test, y_train, y_test = train_test_split(X_std, y, test_size=0.3, random_state=0)
X_train.shape,X_test.shape,X.shape<jupyter_output><empty_output><jupyter_text>Making lr1 as normal logistic regression<jupyter_code>lr1 = LogisticRegression()
lr1.fit(X_train, y_train)
#Mean accuracy of lr1 model
lr1.score(X_test,y_test)<jupyter_output><empty_output><jupyter_text>Making lr2 with solver newton method<jupyter_code>lr2 = LogisticRegression(random_state=0,solver='newton-cg')
lr2.fit(X_train,y_train)
#Mean accuracy of lr2 model
lr2.score(X_test,y_test)<jupyter_output><empty_output><jupyter_text>Plotting the decision boundary<jupyter_code>def plot_decision_regions(X, y, classifier, test_idx=None, resolution=0.02):
   # setup marker generator and color map
   markers = ('s', 'x', 'o', '^', 'v')
   colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
   cmap = ListedColormap(colors[:len(np.unique(y))])

   # plot the decision surface
   x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
   x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
   xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
   np.arange(x2_min, x2_max, resolution))
   Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
   Z = Z.reshape(xx1.shape)
   plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)
   plt.xlim(xx1.min(), xx1.max())
   plt.ylim(xx2.min(), xx2.max())

   # plot all samples
   X_test, y_test = X[test_idx, :], y[test_idx]
   for idx, cl in enumerate(np.unique(y)):
      plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],
               alpha=0.8, c=cmap(idx),
               marker=markers[idx], label=cl)
   # highlight test samples
   if test_idx:
      X_test, y_test = X[test_idx, :], y[test_idx]
      plt.scatter(X_test[:, 0], X_test[:, 1], c='',
               alpha=1.0, linewidth=1, marker='o',
               s=55, label='test set')
X_combined = np.vstack((X_train, X_test))
y_combined = np.hstack((y_train, y_test))
plot_decision_regions(X_combined,
                      y_combined, classifier=lr1,
                      test_idx=range(12219,17457))
y_pred = lr1.predict(X_test)
print(metrics.accuracy_score(y_test,y_pred,normalize=False), 'correctly labelled out of',X_test.shape[0])<jupyter_output>4191 correctly labelled out of 5238
{'fraction_non_alphanumeric': 0.14309728934650137, 'fraction_numerical': 0.01933179239335995, 'mean_word_length': 5.7136812411847675, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '626123', 'license': 'Apache-2.0', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1718, 'n_words': 329, 'path': 'Labs/Lab2/Romil/adult_Log_reg_newton.ipynb', 'repo_name': 'heenalsapovadia/ml_practices_2018', 'size': '4759'}