<jupyter_start><jupyter_text>GET DATA<jupyter_code># read data
train_df = pd.read_csv('data/mix/data41mix_train.csv')
test_df = pd.read_csv('data/mix/data41mix_test.csv')
# label_dict={0:1,1:6,2:11,3:13,4:15,5:17,6:20,7:22,8:25,9:36}

class_size=41
label_dict={}
for i in range(class_size):
    label_dict[i]=i<jupyter_output><empty_output><jupyter_text>PREPROCESS DATA<jupyter_code># initialize preprocess class
preprocess = PreprocessData()
# # split data, normalize, shuffle
X_train, y_train = preprocess.preprocess_data(train_df, normalize=False)
X_test, y_test = preprocess.preprocess_data(test_df, normalize=False)
# new_test=test_df.loc[test_df['# target'].isin([1, 6, 11, 13, 15, 17, 20, 22, 25, 36])]
scaler = preprocessing.StandardScaler().fit(X_train)
X_test=scaler.transform(X_test)
print('Size of training matrix:', X_train.shape)
print('Size of testing matrix:', X_test.shape)
# get unique labesl
unique_words = set(y_test)
print(unique_words)<jupyter_output>{0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0}<jupyter_text>Model Class<jupyter_code># class to handle NN processing
class NNTrainer(object):  
    '''
    Parameters
    ----------
    
    layer_activation: activation function for input and hidden layers
    covariance_type: activation function of end layer
    input_n_cols: numver of columns of input layer
    optimizer_function: optimization function
    loss_function: loss functions
    metrics_v: metric for evaluation result
    epochs_n: number of epoches to update train weights
    batch_size_n: batch size of fitted data
    validation_split_n: ratio of validation split in traning 
    
    choice of parameters depends on the data. 
    '''
    
    def __init__(self, layer_activation='relu', end_layer_activation='softmax',input_n_cols=2808,
                 optimizer_function='adam',loss_function='categorical_crossentropy',metrics_v='accuracy',
                epochs_n=15, batch_size_n=32, validation_split_n=0.2):
        
        # initialize variables
        self.layer_activation = layer_activation
        self.end_layer_activation = end_layer_activation
        self.input_n_cols = input_n_cols
        self.optimizer_function = optimizer_function
        self.loss_function = loss_function
        self.metrics_v = metrics_v
        self.epochs_n = epochs_n
        self.batch_size_n =batch_size_n
        self.validation_split_n = validation_split_n
        
        
        # define model
        self.model = Sequential()
        #add layers to model and initialize
        self.model.add(Dense(50, activation=self.layer_activation, input_shape=(self.input_n_cols,)))
        self.model.add(Dropout(0.2))
#         self.model.add(Dense(200, activation=self.layer_activation, input_shape=(self.input_n_cols,)))
#         self.model.add(Dense(200, activation=self.layer_activation, input_shape=(self.input_n_cols,)))

        self.model.add(Dense(2, activation=self.end_layer_activation))
        
        # compile model
        self.model.compile(optimizer=self.optimizer_function, 
                           loss=self.loss_function,metrics=[self.metrics_v])
            
    #train mode
    def train(self, X_train, y_train):
        # ingonre divisin by 0
        # np.seterr(all='ignore') 
        #train model
        self.model.fit(X_train, y_train, epochs=self.epochs_n,
                       batch_size=self.batch_size_n,validation_split=self.validation_split_n, verbose=0)
         
    # run the model on new data and get score
    def predict_probability(self, X_test):
        scores = self.model.predict_proba(X_test)
        return scores

    
    # return model
    def model_evaluate(self, X_test, y_test):
        scores = self.model.evaluate(X_test, y_test, verbose=1)
        return scores[1]*100
    
    # return model
    def get_nn_model():
        return self.model
    
    def save_modle(self, folder_name, model_name):
        # Creates a HDF5 file 'my_model.h5'
        self.model.save('models/{path}/{model}.h5'.format(path=folder_name,model=model_name))
model_group = {}
prob_group = {}
prob_group_2 = {}
for i in range(len(label_dict)):
    prob_group[i]=[]
    prob_group_2[i]=[]
prob_group
for i in range(0,len(label_dict)):
    
    for j in range(i+1, len(label_dict)):
        
        save_label = str(label_dict[i])+"-"+str(label_dict[j])
#         print(save_label)
                
        
        new_train_df=train_df.loc[train_df['# target'].isin([label_dict[i],label_dict[j]])]

        X_train, y_train = preprocess.preprocess_data(new_train_df, normalize=False)
        X_train=scaler.transform(X_train)
        
        target = np.zeros((len(y_train),2),dtype=int)
        
        for y_t_i,y_l_i in enumerate(y_train):
            
            if(int(y_l_i)==int(label_dict[i])):
                target[y_t_i][0]=1
            else: 
                target[y_t_i][1]=1
                
        
        # train model
        nn_trainer = NNTrainer()
        nn_trainer.train(X_train, target)

        nn_trainer.save_modle(folder_name, "{}".format(save_label))

        # append model
        model_group[save_label]=nn_trainer
        
        nn_trainer = None
# for i in range(0,10):
    
#     logprob = np.array([(model[0].predict_probability(X_test)[:,1],model[0].predict_probability(X_test)[:,0])[i<=model_i] for model_i, model in enumerate(model_group[i])]) ## prob of one gorup
#     prob_group[i] = logprob
   
 def atoi(text):
            return int(text) if str(text).isdigit() else text

def natural_keys(text):
    return [ atoi(c) for c in re.split('(\d+)', int(text) if str(text).isdigit() else text) ]

models=[]

files_list = os.listdir('models/1_vs_1/mix')
files_list.sort(key=natural_keys)

for w in files_list:
# check wheter files is .h5 or no
    if (w.find('.h5')!=-1):
        model_name=w
        model_group[model_name[:-3]] = models.append(load_model('models/1_vs_1/mix'+'/{model}'.format(model=model_name)))
for i in range(0,len(label_dict)):
    for j in range(i+1, len(label_dict)):
        
        save_label = str(label_dict[i])+"-"+str(label_dict[j])
        model=model_group[save_label]
        
        logprob_1=model.predict_proba(X_test)[:,0]
        logprob_2=model.predict_proba(X_test)[:,1]
        
        prob_group[i].append(logprob_1)
        prob_group[j].append(logprob_2)
        
        prob_group_2[i].append(logprob_2)
        prob_group_2[j].append(logprob_1)
        
for i in range(len(label_dict)):
    prob_group[i]=np.array(prob_group[i]).T
results=np.array

mean_l = []
max_l = []

mean_l_2 = []
min_l_2 = []
for i in range(0,len(label_dict)):
    mean_l.append(np.mean(prob_group[i],axis=1))
    max_l.append(np.max(prob_group[i],axis=1))
    
    mean_l_2.append(np.mean(prob_group_2[i],axis=1))
    min_l_2.append(np.min(prob_group_2[i],axis=1))

result_mean=np.vstack(mean_l).T
result_max=np.vstack(max_l).T

result_mean_2=np.vstack(mean_l).T
result_min_2=np.vstack(max_l).T

'''
how to evaluate evaluate model:
    select the ones with larget prob.
    if prob of ones are euqal for any model then select the model with min others prob
'''
logprob_1=result_mean
logprob_2=result_mean_2
result=np.zeros((y_test.shape[0]), dtype=int)

for i in range(y_test.shape[0]):
    max_array=[]
    max_n=-100
    idx=1
    max_min=100
    for j in range(len(label_dict)):
        max_array.append(logprob_1[i][j])
        if logprob_1[i][j]>max_n:
            max_n=logprob_1[i][j]
            max_min=logprob_2[i][j]
            idx=j
            
        if logprob_1[i][j]==max_n and logprob_2[i][j]<max_min:
            max_n=logprob_1[i][j]
            max_min=logprob_2[i][j]
            idx=j
       
    #sort max array
    max_array.sort()
    
    # compare result with the actual labels
    if(int(y_test[i])==int(label_dict[idx]) and max_array[-1]>=.0 and max_array[-1]-max_array[-2]>=0.0):
        result[i]=1
        
        
overall_acc = np.mean(result)*100
print('Overall unseen test accuracy: %.4f percent' % overall_acc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13028682547399126, 'fraction_numerical': 0.03354399611084103, 'mean_word_length': 3.1103896103896105, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '438016', 'license': 'MIT', 'n_tokens_mistral': 3146, 'n_tokens_neox': 2985, 'n_words': 521, 'path': 'src/models/one_vs_one.ipynb', 'repo_name': 'anaramirli/Speech-Alignment', 'size': '8228'}