<jupyter_start><jupyter_text>Setup<jupyter_code># Libraries
import sklearn
import tensorflow as tf
from tensorflow import keras
import numpy as np
import pandas as pd
import os

# Seeds
np.random.seed(42)
tf.random.set_seed(42)

# Plotting
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)
from google.colab import drive
drive.mount('/content/drive/')
# Replace this with your own root
project_root_dir = 'your_drive_here'
aqi_dir = os.path.join(project_root_dir, 'aqi_data', 'final_data')
experiments_dir = os.path.join(project_root_dir, 'experiments')
img_dir = os.path.join(project_root_dir, 'img_data')<jupyter_output><empty_output><jupyter_text>Simple pixel-level softmax  PreliminariesCreate test and training sets<jupyter_code># Load our train, validation, and test datasets
train = np.load(os.path.join(experiments_dir, 'train.npz'))
valid = np.load(os.path.join(experiments_dir, 'valid.npz'))
test = np.load(os.path.join(experiments_dir, 'test.npz'))
# Compute the weights that will help with class imbalance
from collections import Counter
item_ct = Counter(train['y'])
max_ct = float(max(item_ct.values()))
class_wts = {class_id: max_ct / num_img for class_id, num_img in item_ct.items()}
# Center and standardize 
x_mean = train['x'].mean(axis = 0, keepdims = True)
x_std = train['x'].std(axis = 0, keepdims = True) + 1e-7
x_train = (train['x'] - x_mean) / x_std
x_valid = (valid['x'] - x_mean) / x_std
# Flatten the vector 
x_train_flattened = x_train.reshape(x_train.shape[0], -1)
x_valid_flattened = x_valid.reshape(x_valid.shape[0], -1)
# Since we are limited in RAM in colab, delete the old structure
# and garbage collect
from gc import collect
del x_train, x_valid
collect()
# Put the validation and test sets together
x_all = np.concatenate((x_train_flattened, x_valid_flattened))
y_all = np.concatenate((train['y'], valid['y']))
collect()
del x_train_flattened, x_valid_flattened
x_all.shape<jupyter_output><empty_output><jupyter_text>SGD Classifier  Grid searchPerform a grid search to find the best hyperparameter for the regularization and penalty<jupyter_code>from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV

param_grid = [{'alpha' : [0.0001, 0.001, 0.01, 0.1, 1, 10, 100]}, # try over various values of regularization
              {'l1_ratio' : [0, 0.25, 0.5, 0.75, 1]}] # try different mixtures of L1 and L2 penalty

sgd_clf = SGDClassifier(random_state = 42, class_weight = class_wts, penalty = 'elasticnet')
sgd_clf = GridSearchCV(sgd_clf, param_grid, cv = 3, return_train_score = True)
sgd_clf.fit(x_all, y_all)<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># Standardize the test set
x_test = (test['x'] - x_mean) / x_std
x_test_flattened = x_test.reshape(x_test.shape[0], -1)
x_test_flattened.shape
collect()
del x_test
# Create a directory to save results 
directory_name = '11-4-20_SGDClf_classWts_elastic_net'
directory_name = os.path.join(experiments_dir, 'model_results', directory_name)
os.mkdir(directory_name)
x_test = (test['x'] - x_mean) / x_std
x_test_flattened = x_test.reshape(x_valid.shape[0], -1)

test_preds = sgd_clf.predict(x_test_flattened)
test_accuracy = accuracy_score(test['y'], test_preds)
test_accuracy
# Visualize confusion matrix
ax = plt.subplot()
test_cf_matrix = confusion_matrix(test['y'], preds)
sns.heatmap(test_cf_matrix, annot = True, ax = ax, cmap = 'Blues')
plt.title('Confusion matrix: test set')
plt.savefig(os.path.join(directory_name, 'test_set_cf_matrix.png'))
plt.show()
# Save results to our directory
eval_df = pd.DataFrame(columns = ['set', 'loss', 'accuracy'])
eval_df.loc[0] = ['test', 'hinge_loss', test_accuracy]
eval_df.to_csv(os.path.join(directory_name, 'eval_df.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12457999483070561, 'fraction_numerical': 0.016541742052209873, 'mean_word_length': 6.428023032629558, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '214666', 'license': 'MIT', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1429, 'n_words': 333, 'path': 'final_SVM_modeling.ipynb', 'repo_name': 'jakee417/vision_AQI_prediction', 'size': '3869'}