<jupyter_start><jupyter_text>Load data<jupyter_code>df = pd.read_csv('../../Data/Chapter_1_cleaned_data.csv')<jupyter_output><empty_output><jupyter_text>Examining the Relationships between Features and the Response<jupyter_code>features_response = df.columns.tolist()
features_response[:5]
features_response[-5:]
items_to_remove = ['ID', 'SEX',
                   'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5', 'PAY_6',
                   'EDUCATION_CAT',
                   'graduate school', 'high school', 'none',
                   'others', 'university']
example_list_comp = [item for item in range(5)]
example_list_comp
features_response = [item for item in features_response if item not in items_to_remove]
features_response
corr = df[features_response].corr()
corr.iloc[0:5,0:5]
mpl.rcParams['figure.dpi'] = 400 #high res figures
sns.heatmap(corr, 
            xticklabels=corr.columns.values,
            yticklabels=corr.columns.values,
            center=0)
n_points = 500
np.random.seed(seed=2)
X = np.random.uniform(low=0.0, high=10.0, size=(n_points,))
slope = 0.25
noise = 0.2
y=[]
y.append( slope * X + np.random.normal(loc=0.0, scale=noise, size=(n_points,)) )
y.append( -slope * X + np.random.normal(loc=0.0, scale=noise, size=(n_points,)) )
y.append( np.random.normal(loc=0.0, scale=noise, size=(n_points,)) )
y.append( np.sin(X/(2)*np.pi) + np.random.normal(loc=0.0, scale=noise, size=(n_points,)) )
np.corrcoef(X, y[3])
mpl.rcParams['figure.dpi'] = 200
fig_h, ax_h = plt.subplots(2,2)
counter = 0
correlations = []
titles = ['A', 'B', 'C', 'D']
for axis in ax_h.flatten():
    axis.scatter(X,y[counter],s=0.5)
    axis.set_xticks([])
    axis.set_yticks([])
    axis.set_title(titles[counter])
#     axis.set_title( r'$\rho = ${:0.2f}'.format(np.corrcoef(X, y[counter])[0,1]))
    correlations.append(np.corrcoef(X, y[counter])[0,1])
    counter+=1
correlations<jupyter_output><empty_output><jupyter_text>Exercise 11: F-test and Univariate Feature Selection<jupyter_code>X = df[features_response].iloc[:,:-1].values
y = df[features_response].iloc[:,-1].values
print(X.shape, y.shape)
from sklearn.feature_selection import f_classif
[f_stat, f_p_value] = f_classif(X, y)
f_test_df = pd.DataFrame({'Feature':features_response[:-1],
                          'F statistic':f_stat,
                          'p value':f_p_value})
f_test_df.sort_values('p value')
from sklearn.feature_selection import SelectPercentile
selector = SelectPercentile(f_classif, percentile=20)
selector.fit(X, y)
best_feature_ix = selector.get_support()
best_feature_ix
features = features_response[:-1]
best_features = [features[counter]
                 for counter in range(len(features))
                 if best_feature_ix[counter]]
best_features<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15804701627486437, 'fraction_numerical': 0.02314647377938517, 'mean_word_length': 4.319230769230769, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '202671', 'license': 'MIT', 'n_tokens_mistral': 1064, 'n_tokens_neox': 1015, 'n_words': 174, 'path': 'Chapter03/Exercise01/Exercise01.ipynb', 'repo_name': 'arifmudi/Data-Science-Projects-with-Python-Second-Ed', 'size': '2765'}