<jupyter_start><jupyter_text>import data & feature creation* input: file* output: df_features, df_label Data is fromhttps://www.eecs.qmul.ac.uk/mmv/datasets/deap/readme.htmlprepKnow1. https://pandas.pydata.org/pandas-docs/stable/generated/pandas.cut.html<jupyter_code>import _pickle as cPickle
x = cPickle.load(open('data/data_preprocessed_python/s01.dat', 'rb'),encoding='bytes')
x.keys()
arr_x = x[b'data']
y?
y.shape
x[b'labels']
import pandas as pd
df_label = pd.DataFrame(x[b'labels'])

df_label.columns = ['valence', 'arousal', 'dominance', 'liking']

df_label.drop(['dominance', 'liking'],axis=1,inplace=True)

df_label.valence = pd.cut(df_label.valence,bins=2,right=[0,5,10],labels=['high', 'low'])
df_label.arousal = pd.cut(df_label.arousal,bins=2,right=[0,5,10],labels=['high', 'low'])

df_label
arr_x = x[b'data']<jupyter_output><empty_output><jupyter_text>extract liststats1. mean2. median3. max4. min5. std dev6. variance7. range8. skewness9. kurtosisfrom1. 0:806, 10 batches4. all<jupyter_code>arr_x[0,:,:].shape
import scipy
from scipy.stats import skew, kurtosis
import numpy as np
STEPS = np.linspace(0,arr_x.shape[2]-1,11).astype(int); STEPS

ops = [np.mean, np.median, np.max, np.min ,np.std, np.var, skew, kurtosis]
# don't forget range
np.mean(arr_x, axis=2).shape
a = np.stack([op(arr_x, axis=2) for op in ops], axis=2);a.shape
A = a[:,:,2]-a[:,:,3]
A.shape
# add range
a = np.dstack((a,a[:,:,2]-a[:,:,3]))
a1.shape
STEPS[1:]
b = []
for start,end in zip(STEPS.tolist(),STEPS[1:].tolist()):
    ran = np.stack([op(arr_x[:,:,start:end], axis=2) for op in ops], axis=2)
    ran = np.dstack((ran,ran[:,:,2]-ran[:,:,3]))
    b.append(ran)


b = np.concatenate(b,axis=2)
b.shape
PAT = 1
list(arr_x.shape[:2])+[1]
d = np.full(list(arr_x.shape[:2])+[1],PAT)
c = np.reshape(np.repeat(np.arange(arr_x.shape[0]), arr_x.shape[1]),list(arr_x.shape[:2])+[1])
features = np.concatenate((a,b,c,d),axis=2)
features.shape
df_label = pd.get_dummies(df_label)
df_label<jupyter_output><empty_output><jupyter_text>applied for each file<jupyter_code>from glob import glob
import os, re
files = glob('data/data_preprocessed_python/*.dat')
PAT = re.search(r"\d+",os.path.basename(files[0]))
int(PAT.group(0))
int('01')
def readDEAPFile(file):
    with open(file, 'rb') as f:
        x = cPickle.load(f,encoding='bytes')
    return x[b'data'], x[b'labels']
    
def processDEAP(file):
    PAT = re.search(r"\d+",os.path.basename(file))
    PAT = int(PAT.group(0))
    
    arr_x, y = readDEAPFile(file)
    
    # label process
    df_label = pd.DataFrame(y)

    df_label.columns = ['valence', 'arousal', 'dominance', 'liking']

    df_label.drop(['dominance', 'liking'],axis=1,inplace=True)

    df_label.valence = pd.cut(df_label.valence,bins=2,right=[0,5,10],labels=['high', 'low'])
    df_label.arousal = pd.cut(df_label.arousal,bins=2,right=[0,5,10],labels=['high', 'low'])
    df_label = pd.get_dummies(df_label)
    
    
    # features process
    STEPS = np.linspace(0,arr_x.shape[2]-1,11).astype(int); STEPS

    ops = [np.mean, np.median, np.max, np.min ,np.std, np.var, skew, kurtosis] # not include range
    
    # features process: all
    a = np.stack([op(arr_x, axis=2) for op in ops], axis=2)
    a = np.dstack((a,a[:,:,2]-a[:,:,3]))
    
    # features process: the 10-batch
    b = []
    for start,end in zip(STEPS.tolist(),STEPS[1:].tolist()):
        ran = np.stack([op(arr_x[:,:,start:end], axis=2) for op in ops], axis=2)
        ran = np.dstack((ran,ran[:,:,2]-ran[:,:,3]))
        b.append(ran)
    b = np.concatenate(b,axis=2)
    
    # features process: identities
    c = np.reshape(np.repeat(np.arange(arr_x.shape[0]), arr_x.shape[1]),list(arr_x.shape[:2])+[1])
    d = np.full(list(arr_x.shape[:2])+[1],PAT)
    
    features = np.concatenate((a,b,c,d),axis=2)
    
    return features, df_label
xs = []
dfs_y = []
for file in files:
    x, df_y = processDEAP(file)
    xs.append(x)
    dfs_y.append(df_y)
xs = np.stack(xs)
xs.shape # paticipant, experiment, ch, data
ys = [df.values for df in dfs_y]
ys = np.stack(ys)
ys.shape # paticipant, experiment, label<jupyter_output><empty_output><jupyter_text>save and load<jupyter_code>import pickle
with open('data/x.dat',mode='wb') as f:
    pickle.dump(xs, f)
with open('data/y.dat',mode='wb') as f:
    pickle.dump(ys, f)
with open('data/x.dat', mode='rb') as f:
    xs = pickle.load(f)
with open('data/y.dat', mode='rb') as f:
    ys = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Create the network<jupyter_code>import keras
from keras.models import Sequential
from keras.layers import Conv2D, Flatten, Dense, Dropout, MaxPool2D
from keras.optimizers import SGD
batchSize = 50
nbClasses = 2
nbEpoch = 5
CH, DATA = 40, 101
nbFilters = 100
nbConv = 3
NEpoch = 50
model = Sequential()

model.add(Convolution2D(nbFilters, (nbConv, nbConv), input_shape=(1,CH,DATA), activation='tanh', data_format='channels_first'))
model.add(Convolution2D(nbFilters, (nbConv, nbConv), activation='tanh'))
model.add(MaxPool2D())
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128, activation='tanh'))
model.add(Dropout(0.25))
model.add(Dense(nbClasses, activation='softplus'))

sgd = SGD(lr=0.00001, decay= 1e-6, momentum= 0.9, nesterov=True)

model.compile(sgd, 'categorical_crossentropy')


model.summary()
pat, exp, ch ,data = xs.shape
X = xs.reshape(pat*exp, 1, ch, data); X.shape
pat, exp, label = ys.shape
Y = ys.reshape(pat*exp,-1)[:,:2]; Y.shape
accAll = []
for _ in range(NEpoch):
    model.fit(X,Y,batch_size=batchSize, epochs=nbEpoch, validation_split=1/32)
ys.shape
model.evaluate(X,Y)
model.save('originalNet.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19056904804112745, 'fraction_numerical': 0.026236482893104058, 'mean_word_length': 5.014925373134329, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '324330', 'license': 'MIT', 'n_tokens_mistral': 2446, 'n_tokens_neox': 2351, 'n_words': 412, 'path': 'Replica.ipynb', 'repo_name': 'wasdee/EmoRegNet', 'size': '5641'}