<jupyter_start><jupyter_text>Testing linear classifiers on top of PredNet representations<jupyter_code>def get_features(feat_dir, layer_id, i):
    """
    get activations of a particular DNN layer for a particular image

    Input:
    feat_dir: Directory containing activations generated using generate_features.py
    layer_id: layer name
    i: image index

    Output:
    flattened activations
    """
    activations = glob.glob(feat_dir + "/*" + ".mat")
    activations.sort()
    feat = sio.loadmat(activations[i])[layer_id]
    return feat.ravel()

feat = get_features('./prednet/feats/92images/prednet_moments__representation__4_layers_epoch_1/', 'rep_all', 0)
feat.shape
labels = pd.read_csv('92_images.csv')

# feature_dir = './prednet/feats/92images/prednet_kitti_finetuned_moments__representation__10c/'
# feature_dir = './feats/92images_feats/alexnet/'
feature_dir = './prednet/feats/92images/prednet_moments__representation__4_layers_epoch_1/'
train_x = [get_features(feature_dir, 'rep_all', i) for i in range(len(labels))]
train_x = np.array(train_x)
train_y = np.array(labels.animate)

train_x.shape, train_y.shape
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

#pca = PCA(n_components=45)
#x_pca = pca.fit(train_x).transform(train_x)

clf = svm.LinearSVC()
scaler = StandardScaler()
train_x_scaled = scaler.fit_transform(train_x)
skf = StratifiedKFold(n_splits=4)
scores = cross_val_score(clf, train_x_scaled, train_y, cv=skf)
scores, scores.mean()
len(labels[labels.animate == 1]) / len(labels)
def evaluate(model_dir, data_x, data_y, model_type='linear'):
    
    # load model
    model_path = os.path.join(model_dir, model_type + '.pkl')
    model = joblib.load(model_path)
    
    scaler_path = os.path.join(model_dir, model_type + '_scaler.pkl')
    scaler = None
    if os.path.exists(scaler_path):
        scaler = joblib.load(scaler_path)
    
    preds = model.predict(data_x)
    acc = accuracy_score(data_y, preds)
    metric_str = 'acc: {}'.format(acc)
        
    print('Test {}'.format(metric_str))
    #f = open(os.path.join(results_dir, 'test.txt'), 'w')
    #f.write('Test results:\n')
    #f.write(metric_str)
    #f.close()
    
classifier_path = './classifier/models/linear__prednet_kitti_finetuned_moments_10c__2c_easy/'
evaluate(classifier_path, train_x, train_y)<jupyter_output>Test acc: 0.510869565217
{'fraction_non_alphanumeric': 0.12085210979106924, 'fraction_numerical': 0.015157722244981565, 'mean_word_length': 4.800475059382423, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '641744', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 924, 'n_tokens_neox': 878, 'n_words': 156, 'path': 'prednet/train_linear.ipynb', 'repo_name': 'thefonseca/algonauts', 'size': '2441'}