<jupyter_start><jupyter_text>Income Classification<jupyter_code>#Step 1 - Loading the dataset
import pandas as pd
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
#Step 2 - Univariate Analysis
#Let's start by exploring each variable one by one
#datatype (to see if its categorical or continous var)
train.dtypes
#for continous variables
train.describe()
#for categorical vars
#first lets a list of all categorical variables
cat_var = train.dtypes.loc[train.dtypes == 'object'].index
print cat_var
#next, determine the unique values in each column using apply function
train[cat_var].apply(lambda x: len(x.unique()))
#analyzing race (print counts in each category)
train['Se'].value_counts()
#for percentages
train['Race'].value_counts()/train.shape[0]
train.shape
train['Native.Country'].value_counts()
train['Native.Country'].value_counts()/train.shape[0]
#Now lets start with Multivariate analysis
#Even here, Categorical vs Continous is a real deal. There are three combinations
#1. categorical - categorical
#2. continuous - continuous
#3. categorical - continuous

#First Cat-Cat. We'll use cross tabulation for this. Between "Sex" & "Income group"

cross_tab = pd.crosstab(train['Sex'],train['Income.Group'],margins=True)
cross_tab
#plot it using stacked chart
%matplotlib inline
cross_tab.iloc[:-1,:-1].plot(kind='bar',stacked=True,color=['red','blue'],grid=False)
#This is for education
#Round numbers are not very intuitive, so let us calc the percentages and then plot
ct = cross_tab
def percConvert(ser):
    return ser/float(ser[-1])
ct2 = ct.apply(percConvert, axis=1)
ct2.iloc[:-1,:-1].plot(kind='bar',stacked=True,color=['green','brown'],grid=False)
#next, lets do other types. both continous -
train.plot('Age','Hours.Per.Week',kind='scatter')
#this didnt really show us anything interesting
#lets move on to the next type. Category-Continous combination
#so lets rewind, we use bar charts for (i) type, plots for (ii) and box-plots for (iii)

#here lets plot a box plot

train.boxplot(column='Hours.Per.Week', by='Sex')
#Step 3 - Missing value treatment. Solve it using imputation methods
#Checking missing values in training data
train.apply(lambda x: sum(x.isnull()))
#Similiarly check for Testing data
test.apply(lambda x: sum(x.isnull()))
#So you can notice that all 3 of the features are categorical.
#We can solve this by imputing it with mode values

from scipy.stats import mode

mode(train['Workclass']).mode[0]
#lets impute the shit out of it
var_to_impute = ['Workclass','Occupation','Native.Country']
for var in var_to_impute:
    train[var].fillna(mode(train[var]).mode[0], inplace=True)
    test[var].fillna(mode(test[var]).mode[0], inplace=True)
    
train.apply(lambda x: sum(x.isnull()))
test.apply(lambda x: sum(x.isnull()))
#Hurray!
#Step 4 - Outlier Treatment
#Best way to check a outlier is by making a scatter plot
%matplotlib inline
train.plot('ID','Age',kind='scatter')
%matplotlib inline
train.plot('ID','Hours.Per.Week',kind='scatter')
#Step 5 - Variable transformation. 
#it means adding new var, or modifying existing var in order to make our model accurate

#lets that workclass var and explore that

train['Workclass'].value_counts()/train.shape[0]
#here we can see that below 0.05 can be combined to form a higher percentage 

categories_to_combine = ['State-gov','Self-emp-inc','Federal-gov','Without-pay','Never-worked']
#Now run a loop and combine the values of these features into one feature called "Others"

for cat in categories_to_combine:
    train['Workclass'].replace({cat:'Others'},inplace=True)
    test['Workclass'].replace({cat:'Others'},inplace=True)
    
#Check if the changes happened
train['Workclass'].value_counts()/train.shape[0]
#Note that combining is not the best possible techniqe for solving the problem of high cardinality, i.e. high number of unique values.
#Bingo. We did good. But now we have to repeat this for all the features. We can do this using a loop
#Combing the rest
#Step 5.1 - Make a list of variables to combine
categorical_variables = list(train.dtypes.loc[train.dtypes=='object'].index)
categorical_variables
#removing workclass
categorical_variables = categorical_variables[1:]
categorical_variables


#Step 5.2 - check current no of unique values
train[categorical_variables].apply(lambda x: len(x.unique()))
#Step 5.3 - Running a loop and combining categories

for column in categorical_variables:
    #determine the cat to combine
    frq = train[column].value_counts()/train.shape[0]
    categories_to_combine = frq.loc[frq.values < 0.05].index
    
    #loop over these features and combine them as "Others"
    for cat in categories_to_combine:
        train[column].replace({cat:'Others'},inplace=True)
        test[column].replace({cat:'Others'},inplace=True)
    

#Check teh results
train[categorical_variables].apply(lambda x: len(x.unique()))
#Checking the Marital.Status. You can see that unique var reduced from 7 to 4
train['Marital.Status'].value_counts()/train.shape[0]
##Done with all the EDA and Dicing-Slicing. Lets get down to business now
##Modeling time!

#For modeling using sklearn in Pandas, we have to adopt the following steps:
#Data Preprocessing: sklearn accepts only numeric data so we've to convert text to numbers
#Training model: fit the model on training data
#Making predictions: use the model to make predictions on test data
#Analyze results: compare the train and test accuracy and derive insights required to improve the model
#Step 1 - Data Preprocessing. Conv text to numbers. 
#Label encoder is used to conv categories to num

from sklearn.preprocessing import LabelEncoder
categorical_variables = train.dtypes.loc[train.dtypes=='object'].index
categorical_variables
#Conv em using Label Encoder
le = LabelEncoder()
for var in categorical_variables:
    train[var] = le.fit_transform(train[var])
    test[var] = le.fit_transform(test[var])
#Checking using dtypes
train.dtypes
#Step 2 - Fit the model
#we are using Decision Tree Classifier for this

from sklearn.tree import DecisionTreeClassifier

#Define predictors - Here it will all vars except ID & Target var

dependent_var = 'Income.Group'
independent_var = [x for x in train.columns if x not in ['ID', dependent_var]]
print independent_var
#We'll have few benchmark parameters
#max_depth = 10
#min_samples_leaf = 100
#max_features = 'sqrt'
#Init the algo

model = DecisionTreeClassifier(max_depth=10,min_samples_leaf=100,max_features='sqrt')

#Fit the algo

model.fit(train[independent_var],train[dependent_var])
#Step 3 - Make predictions 

predictions_train = model.predict(train[independent_var])
predictions_test = model.predict(test[independent_var])

#Step 4 - Analyze results

from sklearn.metrics import accuracy_score
#Determine train error
acc_train = accuracy_score(train[dependent_var],predictions_train)

print "Training accuracy =  %f"%acc_train
#Step 5 - Making Submission

submission=pd.read_csv('sample_submisson.csv')
submission.ID = test.ID
income_group = ['<=50K']*len(test['ID'])

for i in range(len(predictions_test)):
    if predictions_test[i] == 1:
        income_group[i] = '>50K'
submission['Income.Group'] = income_group
submission.to_csv('submission02.csv', index=False)
Score is 0.813<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11016131256031987, 'fraction_numerical': 0.009237556873018062, 'mean_word_length': 5.576609247506799, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '155531', 'license': 'MIT', 'n_tokens_mistral': 2319, 'n_tokens_neox': 2200, 'n_words': 754, 'path': 'K Nearest Neighbor - Income Classification/.ipynb_checkpoints/Income classification-checkpoint.ipynb', 'repo_name': 'warmachine0609/Data-Science-Projects', 'size': '7253'}