<jupyter_start><jupyter_text>ANN to Predict SubjectHow few channels are needed? Setup: Javascript and ipynb stuff<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
%%javascript
utils.load_extension('collapsible_headings/main')
utils.load_extension('hide_input/main')
utils.load_extension('execute_time/ExecuteTime')
utils.load_extension('code_prettify/code_prettify')
utils.load_extension('scroll_down/main')
utils.load_extension('jupyter-js-widgets/extension')<jupyter_output><empty_output><jupyter_text>Load Functions<jupyter_code>from fastai.vision import *
import os
import numpy as np
import pandas as pd
import pickle
path=pathlib.Path('/media/tris/tris_files/DEAP_dataset/DMD')
path<jupyter_output><empty_output><jupyter_text>Load Original Data Labels<jupyter_code>path = '/media/tris/tris_files/DEAP_dataset/data_preprocessed_python/s01.dat'
df_read = pickle.load(open(path, 'rb'), encoding='latin1')
labels=df_read['labels']
df_tmp = pd.DataFrame(labels, columns=['valence','arousal','dominance','liking'])
df=df_tmp
for n in range (2,10):
    path = '/media/tris/tris_files/DEAP_dataset/data_preprocessed_python/s0'+str(n)+'.dat'
    df_read = pickle.load(open(path, 'rb'), encoding='latin1')
    labels=df_read['labels']
    df_tmp = pd.DataFrame(labels, columns=['valence','arousal','dominance','liking'])
    df=df.append(df_tmp, ignore_index=True)
for n in range (10,33):
    path = '/media/tris/tris_files/DEAP_dataset/data_preprocessed_python/s'+str(n)+'.dat'
    df_read = pickle.load(open(path, 'rb'), encoding='latin1')
    labels=df_read['labels']
    df_tmp = pd.DataFrame(labels, columns=['valence','arousal','dominance','liking'])
    df=df.append(df_tmp, ignore_index=True) 
df=df.div(9) #normalize
df.head()<jupyter_output><empty_output><jupyter_text>Set filenames and labels<jupyter_code>filenames=[]
subject_labels=[]
for n in range (0,9):
    for i in range (0,9):
        filename='S'+str(n+1)+'T'+str(i+1)
        filenames.append(filename)
        subject_label=n+1
        subject_labels.append(subject_label)
    for i in range (9,40):
        filename='S'+str(n+1)+'T'+str(i+1)
        filenames.append(filename)
        subject_label=n+1
        subject_labels.append(subject_label)
for n in range (9,32):
    for i in range (0,9):
        filename='S'+str(n+1)+'T'+str(i+1)
        filenames.append(filename)
        subject_label=n+1
        subject_labels.append(subject_label)
    for i in range (9,40):
        filename='S'+str(n+1)+'T'+str(i+1)
        filenames.append(filename)
        subject_label=n+1
        subject_labels.append(subject_label)
df['file_name']=filenames
df['subject_label']=subject_labels
# df=df[['file_name','valence','arousal','dominance','liking']]
df=df[['file_name','subject_label']]
df.tail()
# for i in range(1,33):
#     for ii in range(1,41):
#         os.rename(r'/media/tris/tris_files/github/OoMA-omniscient/data/processed/heatmaps_ind/heatmaps/T'+str(ii)+'S'+str(i)+".png",r'/media/tris/tris_files/github/OoMA-omniscient/data/processed/heatmaps_ind/heatmaps/S'+str(i)+'T'+str(ii)+".png")
# from PIL import Image

# for i in range (0,1280):
    
#     im = Image.open(r"/media/tris/tris_files/github/OoMA-omniscient/data/processed/heatmaps_ind/heatmaps/"+str(df.file_name[i])+".png")
#     im1=im.crop((0,0,1450,1030))
#     im1.save("/media/tris/tris_files/github/OoMA-omniscient/data/processed/heatmaps_ind/heatmaps_re/"+str(df.file_name[i])+".png")
path=pathlib.Path('/media/tris/tris_files/DEAP_dataset/DMD')
path
df.to_csv(path/'labels_proc.csv', index=False)
df = pd.read_csv(path/'labels_proc.csv') #load labels
df.tail()<jupyter_output><empty_output><jupyter_text>Data loader<jupyter_code>src = (ImageList.from_csv(path, 'labels_proc.csv', folder='heatmaps_10ch', suffix='.png')
       .split_by_rand_pct(0.2)
       .label_from_df(cols=['subject_label'],label_cls=CategoryList)
#        .label_from_df(cols=['valence','arousal','dominance','liking'],label_cls=FloatList)
       .transform()
       .databunch(bs=4).normalize())
src.show_batch(rows=4, figsize=(12,9))<jupyter_output><empty_output><jupyter_text>Setup Network<jupyter_code>learn = cnn_learner(src, models.resnet18, metrics=accuracy)
learn.summary()
learn.lr_find()
learn.recorder.plot()<jupyter_output><empty_output><jupyter_text>Train last layers<jupyter_code>lr = 1e-2
learn.fit_one_cycle(8, slice(lr))
learn.recorder.plot_losses()
learn.show_results()<jupyter_output><empty_output><jupyter_text>Train the whole network<jupyter_code>learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
lr = 1e-5
learn.fit_one_cycle(4, slice(lr))
learn.fit_one_cycle(4, slice(lr))<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>df_res=[]
df_res = pd.DataFrame(columns=['no_channels','accuracy'])
info0 = {'no_channels': 2, 'accuracy' : 63}
info1 = {'no_channels': 6, 'accuracy' : 91}
info2 = {'no_channels': 10, 'accuracy' : 99}
info3 = {'no_channels': 15, 'accuracy' : 99}
info4 = {'no_channels': 20, 'accuracy' : 99}
info5 = {'no_channels': 25, 'accuracy' : 99}
info6 = {'no_channels': 30, 'accuracy' : 99}
info7 = {'no_channels': 32, 'accuracy' : 99}
    
df_res = df_res.append([info0, info1, info2, info3, info4, info5, info6, info7], ignore_index=True)
# df_res = df_res.append(info1, ignore_index=True)
df_res
df_res.plot(x='no_channels',y='accuracy',kind='line')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16824865066071096, 'fraction_numerical': 0.02419504932067746, 'mean_word_length': 6.3315143246930425, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '358576', 'license': 'MIT', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2071, 'n_words': 272, 'path': 'src/models/2020-06-11_fingerprint-ML-DMD_05.ipynb', 'repo_name': 'tdgriffith/OoMA-omniscient', 'size': '5373'}