<jupyter_start><jupyter_text>AnalyticsVidhya Workshop<jupyter_code>import pandas as pd
# let's load the training and test data
train = pd.read_csv('data/train.csv')
test = pd.read_csv('data/test.csv')<jupyter_output><empty_output><jupyter_text>Univariate Analysis<jupyter_code>train.dtypes
# continuous variables
train.describe()
# step 1: get a list of categorical variables
categorical_variables = train.dtypes.loc[train.dtypes=='object'].index
print(categorical_variables)
# step 2: use the apply function in python to
# determine the number of unique values in each column
train[categorical_variables].apply(lambda x: len(x.unique()))
# Analyzing Race
# print the counts of each category
train['Race'].value_counts()
# print the percentage of observations in each category
train['Race'].value_counts()/train.shape[0]
# Analyzing Native-Country
# print the counts of each category
train['Native.Country'].value_counts()
# print the percentage of observations in each category
train['Native.Country'].value_counts()/train.shape[0]
q1 = train['Age'].quantile(0.25)
q3 = train['Age'].quantile(0.75)
iqr = q3 - q1
iqr<jupyter_output><empty_output><jupyter_text>Multivariate AnalysisThe type of visualization technique to use depends on the type of each variable. Thus there can be 3 combinations of the type of the 2 variables:1. categorical - categorical2. continuous - continuous3. categorical - continuous  categorical - categorical<jupyter_code># print the cross tabulation
ct = pd.crosstab(train['Sex'], train['Income.Group'], margins=True)
ct
# plot using a stacked chart
%matplotlib inline
ct.iloc[:-1,:-1].plot(kind='bar', stacked=True, color=['red', 'blue'], grid=False)
def perc_convert(ser):
    return ser/float(ser[-1])
ct2 = ct.apply(perc_convert, axis=1)
ct2.iloc[:-1,:-1].plot(kind='bar', stacked=True, color=['red', 'blue'], grid=False)<jupyter_output><empty_output><jupyter_text>continuous - continuous<jupyter_code>train.plot('Age', 'Hours.Per.Week', kind='scatter')<jupyter_output><empty_output><jupyter_text>categorical - continuous<jupyter_code>train.boxplot(column='Hours.Per.Week', by='Sex')
ct2<jupyter_output><empty_output><jupyter_text>Missing Value Treatment  Checking missing values<jupyter_code># checking the missing values in the training data
train.apply(lambda x: sum(x.isnull()))
# checking the missing values in the test data
test.apply(lambda x: sum(x.isnull()))<jupyter_output><empty_output><jupyter_text>Imputation<jupyter_code>from scipy.stats import mode
mode(train['Workclass'].astype('str')).mode[0]
var_to_impute = ['Workclass', 'Occupation', 'Native.Country']
for var in var_to_impute:
    train[var].fillna(mode(train[var].astype('str')).mode[0], inplace=True)
    test[var].fillna(mode(test[var].astype('str')).mode[0], inplace=True)
mode(train['Workclass']).mode[0]
# check again for 'missing values' again to confirm
train.apply(lambda x: sum(x.isnull()))
test.apply(lambda x: sum(x.isnull()))<jupyter_output><empty_output><jupyter_text>Outlier treatment<jupyter_code>train.plot('ID', 'Age', kind='scatter')
train.plot('ID', 'Hours.Per.Week', kind='scatter')<jupyter_output><empty_output><jupyter_text>Variable transformation<jupyter_code>train['Workclass'].value_counts()/train.shape[0]
categories_to_combine = ['State-gov','Self-emp-inc','Federal-gov','Without-pay','Never-worked']
for cat in categories_to_combine:
    train['Workclass'].replace({cat:'Others'},inplace=True)
    test['Workclass'].replace({cat:'Others'},inplace=True)
train['Workclass'].value_counts()/train.shape[0]
test['Workclass'].value_counts()/train.shape[0]
# lets combine the rest of the categorical vairables
# removing 'workclass' because already combined
categorical_variables = categorical_variables[1:]
# current no. of unique values
train[categorical_variables].apply(lambda x: len(x.unique()))
for column in categorical_variables:
    # determine the categories to combine
    frq = train[column].value_counts()/train.shape[0]
    categories_to_combine = frq.loc[frq.values<0.05].index
    
    for cat in categories_to_combine:
        train[column].replace({cat:'Others'},inplace=True)
        test[column].replace({cat:'Others'},inplace=True)
train[categorical_variables].apply(lambda x: len(x.unique()))
test[categorical_variables[:-1]].apply(lambda x: len(x.unique()))
# bins = list(range(0, 101, 5))
# pd.value_counts(pd.cut(train['Hours.Per.Week'], bins))<jupyter_output><empty_output><jupyter_text>Predictive Modeling  Data Preprocessing<jupyter_code>from sklearn.preprocessing import LabelEncoder
categorical_variables = train.dtypes.loc[train.dtypes=='object'].index
categorical_variables
le = LabelEncoder()

for var in categorical_variables:
    train[var] = le.fit_transform(train[var])
    if var in test.columns:
        test[var] = le.fit_transform(test[var])
train.dtypes<jupyter_output><empty_output><jupyter_text>Fit the model<jupyter_code>from sklearn.tree import DecisionTreeClassifier
# define the predictors which will be all
# variables except ID and target
dependent_variable = 'Income.Group'
independent_variables = [x for x in train.columns if x not in ['ID', dependent_variable]]
independent_variables
# initialize the algorithm
model = DecisionTreeClassifier(max_depth=10, min_samples_leaf=50, max_features='sqrt')
#model = DecisionTreeClassifier(max_depth=50,min_samples_leaf=2)

# fit the algorithm
model.fit(train[independent_variables], train[dependent_variable])<jupyter_output><empty_output><jupyter_text>Make predictions<jupyter_code>predictions_train = model.predict(train[independent_variables])
predictions_test = model.predict(test[independent_variables])<jupyter_output><empty_output><jupyter_text>Analyze results<jupyter_code>from sklearn.metrics import accuracy_score
# determine the train error
acc_train = accuracy_score(train[dependent_variable], predictions_train)
"Train accuracy: %f" % acc_train
df = pd.DataFrame(predictions_test, columns=['Income.Group'], index=test['ID'])
df.loc[df['Income.Group'] == 0, 'Income.Group'] = "<=50K"
df.loc[df['Income.Group'] == 1, 'Income.Group'] = ">50K"
df.to_csv('data/sample_submission.csv', index_label='ID', header=['Income.Group'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13971539456662355, 'fraction_numerical': 0.009540750323415266, 'mean_word_length': 7.937861271676301, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '430528', 'license': 'MIT', 'n_tokens_mistral': 2092, 'n_tokens_neox': 2016, 'n_words': 403, 'path': 'AnalyticsVidhya_Workshop.ipynb', 'repo_name': 'vishalkdotcom/AnalyticsVidhya_Workshop', 'size': '6184'}