<jupyter_start><jupyter_text>** Our dataset contains both categorical as well as continuous variables. **<jupyter_code># concatenate train and test set
data = pd.concat((train, test))
# set id as the index of our concatenated dataframe
data = data.set_index('id')
# take the target variable to log domain
data['log_loss'] = data.loss.map(np.log)
def unique_values(data):
    features = data.columns[:-1]
    
    for feat in features:
        print('Feature: {}'.format(feat))
        print('Number of unique values in training set: {}'.format(data[:len(train)][feat].nunique()))
        print('Number of unique values in test set: {}'.format(data[len(train):][feat].nunique()))
        print('\n')
unique_values(data)
data['cont1'].describe()
sns.kdeplot(data.cont1)
sns.regplot(x='cont1', y='log_loss', data=data[:len(train)], fit_reg=False);
continuous_variables = [col for col in data.columns if 'cont' in col]
print('Number of continuous variables are: {}'.format(len(continuous_variables)))
continuous_variables
fig, ax = plt.subplots(nrows=7, ncols=2, figsize=(15, 10), sharey=True)

sns.regplot(x='cont1', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[0][0])
sns.regplot(x='cont2', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[0][1])
sns.regplot(x='cont3', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[1][0])
sns.regplot(x='cont4', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[1][1])
sns.regplot(x='cont5', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[2][0])
sns.regplot(x='cont6', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[2][1])
sns.regplot(x='cont7', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[3][0])
sns.regplot(x='cont8', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[3][1])
sns.regplot(x='cont9', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[4][0])
sns.regplot(x='cont10', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[4][1])
sns.regplot(x='cont11', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[5][0])
sns.regplot(x='cont12', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[5][1])
sns.regplot(x='cont13', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[6][0])
sns.regplot(x='cont14', y='log_loss', data=data[:len(train)], fit_reg=False, ax=ax[6][1]);<jupyter_output><empty_output><jupyter_text>** Summary *** Cont1, Cont6, Cont8, Cont9, Cont10, Cont11, Cont12 has some of the values clubbed at around 0.0 which might need some treatment.* Cont2, Cont3 can be considered as a categorical variable ?* Rest of the continuous variables are not providing much information regarding the target variable. ** Establish Relationships among categorical variables. **<jupyter_code>cat1_cat2 = data[:len(train)].pivot_table(index='cat1', columns='cat2', values='loss', aggfunc='sum', margins=False)
cat1_cat2 / cat1_cat2.sum(axis=0)
data[data.loss < 20000].boxplot(column='loss', by='cat1');
data[data.loss < 20000].boxplot(column='loss', by='cat2');
cat1_cat3 = data[:len(train)].pivot_table(index='cat1', columns='cat3', values='loss', aggfunc='sum', margins=False)
cat1_cat3 / cat1_cat3.sum(axis=0)
data[data.loss < 20000].boxplot(column='loss', by='cat3');
cat2_cat4 = data[:len(train)].pivot_table(index='cat2', columns='cat4', values='loss', aggfunc='sum', margins=False)
cat2_cat4 / cat2_cat4.sum(axis=0)
cat2_cat3 = data[:len(train)].pivot_table(index='cat2', columns='cat3', values='loss', aggfunc='sum', margins=False)
cat2_cat3 / cat2_cat3.sum(axis=0)
cat3_cat4 = data[:len(train)].pivot_table(index='cat3', columns='cat4', values='loss', aggfunc='sum', margins=False)
cat3_cat4 / cat3_cat4.sum(axis=0)
categorical_variables = [col for col in data.columns if 'cat' in col]
print('Number of categorical variables ', len(categorical_variables))
list(combinations(['A', 'B', 'C'], 2))<jupyter_output><empty_output><jupyter_text>** Frequency count for all levels of categorical features. **<jupyter_code>def level_freq(df):
    categorical_columns = df.select_dtypes(include=['object']).columns
    
    for feat in categorical_columns:
        if feat in ['loss', 'id']:
            continue
        print('Feat: {}'.format(feat))
        print('Number of unique values {}'.format(df[feat].nunique()))
        print('Frequency count per level \n')
        print(df[feat].value_counts())
        print('-'*50 + '\n')
level_freq(train)
level_freq(test)
class CategoricalFeatureInteraction(object):
    def __init__(self, df, features):
        self.df       = df
        self.features = features
    
    def capture_interaction(self):
        for (feat1, feat2) in combinations(self.features, 2):
            feat1_feat2 = self.df.pivot_table(index=feat1, columns=feat2, values='loss', aggfunc='sum', margins=False, fill_value=0)
            
            print('Feature Interaction {0}-{1}:\n'.format(feat1, feat2))
            print('-'*50)
            print('\n')
            
            print(feat1_feat2 / feat1_feat2.sum(axis=0))
cat_interaction = CategoricalFeatureInteraction(data[:len(train)], categorical_variables[:10])
cat_interaction.capture_interaction()
data[data.loss < 4e4].boxplot(column='loss', by='cat80')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18542541014879818, 'fraction_numerical': 0.030904235024799696, 'mean_word_length': 6.152796725784447, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '189311', 'license': 'MIT', 'n_tokens_mistral': 1935, 'n_tokens_neox': 1855, 'n_words': 365, 'path': 'notebooks/What_is_data.ipynb', 'repo_name': 'numb3r33/AllStateInsurance', 'size': '5242'}