<jupyter_start><jupyter_text>Glance of Data<jupyter_code># read data
data_raw = pd.read_csv("../data/student-mat.csv", header = 0, delimiter= ";")
data_raw.head()
# summary of categorical data
data_raw.describe(include = [np.object])
# summary of numercial data
data_raw.describe()<jupyter_output><empty_output><jupyter_text>Data preprocessing Drop `G1` and `G2``G1` and `G2` are grades in first and second periods, and they are highly correlated to the target `G3`, so we would like to drop these two features.  Encoding categorical dataThere are 13 binary categorical features and 4 multi-category features. For the binary features, we can convert them to be `0` and `1`. For the multi-categroy features, we shall use _One Hot Encoding_ to create multiple binary features, avoiding the issue of implied order between different categories.<jupyter_code>categ_list = list(data_raw.describe(include = [np.object]))
categ_listB = categ_list[0:5]+categ_list[9:]
categ_listM = categ_list[5:9]

data_new = data_raw.copy()
data_new.drop(['G1', 'G2'], axis=1, inplace=True)

# encoding binary categorical data
ln = LabelEncoder()
for categ in categ_listB:
    ln.fit(data_raw[categ])
    encode = ln.transform(data_raw[categ])
    data_new[categ] = encode

# encoding multi-category features
data = pd.get_dummies(data_new)
data.head()<jupyter_output><empty_output><jupyter_text>Data visualizationHere we briefly visualize the data.<jupyter_code># distribution of the target variable
p1 = sns.distplot(data.G3)
plt.xlabel("Grade", fontsize = 16)
plt.ylabel("Density", fontsize = 16)
plt.show()
figure = p1.get_figure()    
figure.savefig('../results/images/G3.png', dpi=200)<jupyter_output><empty_output><jupyter_text>Feature and Model SelectionHere we perform feature selection<jupyter_code># split data  
X = data.drop("G3", axis=1)
y = data["G3"]
features = list(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1234)<jupyter_output><empty_output><jupyter_text>Training without feature selection<jupyter_code># function to fit and report errors
def fit_and_report(model, X, y, Xv, yv):
    model.fit(X,y)
    mean_squared_err = lambda y, yhat: np.mean((y-yhat)**2)
    errors = [mean_squared_err(y, model.predict(X)), mean_squared_err(yv, model.predict(Xv))]
    print("Train error: %.3f" % errors[0])
    print("Test error: %.3f" % errors[1])
    return errors
# simple fit with linear regression without feature selection
print("Linear regression errors:")
errors = fit_and_report(LinearRegression(), X_train, y_train, X_test, y_test)
# simple fit with ridge without feature selection
print("Ridge errors:")
errors = fit_and_report(Ridge(alpha=1), X_train, y_train, X_test, y_test)
# find the optimal hyperparameters for random forest
parameters = {'max_depth':(range(1,15)), "n_estimators":(20, 30, 40, 50)}
rfr = RandomForestRegressor()
clf = GridSearchCV(rfr, parameters,cv =10)
clf.fit(X_train, y_train)
clf.best_params_
# simple fit with Random Forest regressor without feature selection
print("Random Forest regressor errors:")
errors = fit_and_report(RandomForestRegressor(n_estimators = 30, max_depth=11), X_train, y_train, X_test, y_test)<jupyter_output>Random Forest regressor errors:
Train error: 2.628
Test error: 13.603<jupyter_text>Forward Selection<jupyter_code># function for forward selection
def forward_selection(model, X, y, Xv, yv, d_max = None):
    n, d = X.shape
    if d_max == None:
        d_max = d
    
    # empty list for saving the index of related features
    select_index = []
    # list for the index that left after selecting
    left_index = list(range(d))
    # initialize the mean squared error to inf
    err0 = np.Inf
    
    ii = 0
    # selecting loop
    while ii<d_max:
        # initialized the index that gives the max reduction (-1 is not in the range 0 to 300)
        tmpindex = -1
        # initialized the mean squared error for one iteration
        errv = err0
        # loop for iterating over all left features
        for i in left_index:
            # fit data
            model.fit(X[:,select_index+[i]], y)
            # predict for validation set
            yv_pred = model.predict(Xv[:,select_index+[i]])
            # calcualte error for including feature with index i
            errv_i = np.mean((yv_pred-yv)**2)
            # compare with previous smallest error
            if errv_i < errv:
                # in case this error_i is smaller, then assign them to errv and save the index
                errv = errv_i
                tmpindex = i
        # if haven't find the feature can reduce error, then tmpindex is still -1, then stop the loop
        if tmpindex == -1:
            break
        else:
            # add useful features to lis
            select_index.append(tmpindex)
            left_index.remove(tmpindex)
            err0 = errv
        ii = ii +1
            
    return select_index

def sel_fea(select_index, features):
    selected_features = []

    for index in select_index:
        selected_features.append(features[index])
    return(selected_features)

# select features using forward selection
select_index_FS = forward_selection(LinearRegression(), X_train.as_matrix(), y_train, X_test.as_matrix(), y_test)
print("The index for selected features:", select_index_FS)
print("------------")
print("Selected features:", sel_fea(select_index_FS, features))
# simple fit with linear regression without feature selection
print("Linear regression errors:")
errors = fit_and_report(LinearRegression(), X_train.iloc[:,select_index_FS], y_train, X_test.iloc[:,select_index_FS], y_test)
# simple fit with Random Forest regressor without feature selection
print("Random Forest regressor errors:")
errors = fit_and_report(RandomForestRegressor(n_estimators = 30, max_depth=18), X_train.iloc[:,select_index_FS], y_train, X_test.iloc[:,select_index_FS], y_test)<jupyter_output>Random Forest regressor errors:
Train error: 2.966
Test error: 14.185<jupyter_text>The summary of the selected features using forward selection<jupyter_code>X.iloc[:,select_index_FS].describe()<jupyter_output><empty_output><jupyter_text>L1 regularization<jupyter_code># find the optimal hyperparameters for random forest
parameters = {'alpha':(0.01, 0.05, 0.1, 0.2, 0.25, 0.3, 0.35,0.4,0.5,1,10,100)}
lasso = Lasso()
clf = GridSearchCV(lasso, parameters,cv =10)
clf.fit(X_train, y_train)
clf.best_params_
lasso = Lasso(alpha = 0.35)
lasso.fit(X_train,y_train)
index = np.arange(X_train.shape[1])
print("nonzero indices",index[np.abs(lasso.coef_) > 0.00000000000001])
print("--------------")
print("Selected features:", sel_fea(index[np.abs(lasso.coef_) > 0.00000000000001], features))
# simple fit with Lasso without feature selection
print("Lasso errors:")
errors = fit_and_report(Lasso(alpha = 0.35), X_train, y_train, X_test, y_test)<jupyter_output>Lasso errors:
Train error: 18.093
Test error: 16.847<jupyter_text>Elastic Net<jupyter_code># find the optimal hyperparameters for Elastic Net
parameters = {'alpha':(0.01, 0.05, 0.1, 0.2, 0.25, 0.3, 0.35,0.4,0.5,1,10,100), "l1_ratio":(np.arange(1,10)*0.1)}
en = ElasticNet()
clf = GridSearchCV(en, parameters,cv =10)
clf.fit(X_train, y_train)
clf.best_params_
en = ElasticNet(alpha = 0.25, l1_ratio=0.6)
en.fit(X_train,y_train)
index = np.arange(X_train.shape[1])
print("nonzero indices",index[np.abs(en.coef_) > 0.00000000000001])
print("--------------")
print("Selected features:", sel_fea(index[np.abs(en.coef_) > 0.00000000000001], features))
# simple fit with Lasso without feature selection
print("Lasso errors:")
errors = fit_and_report(ElasticNet(alpha = 0.25, l1_ratio=0.6), X_train, y_train, X_test, y_test)<jupyter_output>Lasso errors:
Train error: 17.047
Test error: 17.098<jupyter_text>Test with SVR using rbf kernelFrom the results above, we see that `RandomForestRegressor()` without feature selection and `LinearRegressor` with forward selection give lower testing errors. Here we extended to another nonlinear regressor `SVR`.   Without Feature Selection<jupyter_code># find the optimal hyperparameters for SVR
parameters = {'C':(0.001,0.01,0.1,1,10,100,1000), 'gamma':(0.001,0.01,0.1,1,10,100,1000)}
svr = SVR()
clf = GridSearchCV(svr, parameters,cv =10)
clf.fit(X_train, y_train)
clf.best_params_
# find the optimal hyperparameters for SVR, finer grid
parameters = {'C':(list(range(1,11))), 'gamma':(np.arange(1,11)*0.01)}
svr = SVR()
clf = GridSearchCV(svr, parameters,cv =10)
clf.fit(X_train, y_train)
clf.best_params_
# simple fit with SVR without feature selection
print("SVR errors:")
errors = fit_and_report(SVR(C = 5, gamma = 0.05), X_train, y_train, X_test, y_test)<jupyter_output>SVR errors:
Train error: 5.351
Test error: 15.444<jupyter_text>Use selected features from forward selection<jupyter_code># find the optimal hyperparameters for SVR, finer grid
parameters = {'C':(list(range(1,11))), 'gamma':(np.arange(1,11)*0.005)}
svr = SVR()
clf = GridSearchCV(svr, parameters,cv =10)
clf.fit(X_train.iloc[:,select_index_FS], y_train)
clf.best_params_
# simple fit with SVR without feature selection
print("SVR errors:")
errors = fit_and_report(SVR(C = 5, gamma = 0.015), X_train.iloc[:,select_index_FS], y_train, X_test.iloc[:,select_index_FS], y_test)<jupyter_output>SVR errors:
Train error: 14.525
Test error: 14.583<jupyter_text>The prediction<jupyter_code># plot random forest predicitons
rfr = RandomForestRegressor(n_estimators = 30, max_depth=11)

rfr.fit(X_train, y_train)

y_pred_train = rfr.predict(X_train)
y_pred_test = rfr.predict(X_test)

fig, ax = plt.subplots(figsize = [10,10])
ax.plot([0, 20], [0,20], "k--", label = "Ideal")
ax.scatter(y_train, y_pred_train, color = "red", label = "Training")
ax.scatter(y_test, y_pred_test, color = "blue",label = "Test")
ax.set_aspect('equal')
plt.xlabel("Grade (true)", fontsize = 16)
plt.ylabel("Grade (Prediction)", fontsize = 16)
plt.title("Random Forest Regression", fontsize = 20)
plt.legend(fontsize = 16)
plt.show()
fig.savefig('../results/images/RandomForest.png', dpi=200)
features = list(data)
importances = rfr.feature_importances_
indices = np.argsort(importances)

features_order = []
for index in indices:
    features_order.append(features[index])

fig = plt.figure(figsize = [10,10])
plt.title('Feature Importances (Random Forest)', fontsize = 20)
plt.barh(range(len(indices)), importances[indices], color='g', align='center')
plt.yticks(range(len(indices)), features_order, fontsize = 13)
plt.xlabel('Relative Importance', fontsize = 16)
plt.show()
fig.savefig('../results/images/RandomForest_importance.png', dpi=300)
# plot linear regression predicitons
lm = LinearRegression()
lm.fit(X_train.iloc[:,select_index_FS], y_train)

y_pred_train = lm.predict(X_train.iloc[:,select_index_FS])
y_pred_test = lm.predict(X_test.iloc[:,select_index_FS])

fig, ax = plt.subplots(figsize = [10,10])
ax.plot([0, 20], [0,20], "k--", label = "Ideal")
ax.scatter(y_train, y_pred_train, color = "red", label = "Training")
ax.scatter(y_test, y_pred_test, color = "blue",label = "Test")
ax.set_aspect('equal')
plt.xlabel("Grade (true)", fontsize = 16)
plt.ylabel("Grade (Prediction)", fontsize = 16)
plt.title("Linear Regression with Forward Selection", fontsize = 20)
plt.legend(fontsize = 16)
plt.show()
fig.savefig('../results/images/LinearRegression.png', dpi=200)
X.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13087159601656825, 'fraction_numerical': 0.03657354366792985, 'mean_word_length': 5.101075268817205, 'pattern_counts': {'":': 2, '<': 45, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '543439', 'license': 'MIT', 'n_tokens_mistral': 4240, 'n_tokens_neox': 3903, 'n_words': 993, 'path': 'src/data_analysis.ipynb', 'repo_name': 'ShunChi100/MachineLearningAnalysis_Student_Performance', 'size': '11347'}