<jupyter_start><jupyter_text>MODEL<jupyter_code>def f1_comp(cf):
    prec = 0
    rec = 0
    f1 = 0
    if (cf[0] + cf[2]) != 0:
        prec = cf[0] / (cf[0] + cf[2])
    if (cf[0] + cf[3]) != 0:
        rec  = cf[0] / (cf[0] + cf[3])
    if (prec + rec) != 0:
        f1   = 2 * (prec * rec / (prec+rec))
        
    return f1, prec, rec
def conf_mat(pred,truth):
    res = [0,0,0,0]
    a = 0
    
    for i in range(len(truth)):
        if truth[i] == 1:
            if truth[i] == pred[i]:
                a = 0
            else:
                a = 2
                
        else:
            if truth[i] == pred[i]:
                a = 1
            else:
                a = 3
                
        res[a] = res[a] + 1
     
    print(res)
    return res
def predict(r_mean,limit):
    res = [0 for i in range(len(r_mean))]

    for i in range(1,len(r_mean)):
        if abs(r_mean[i] - r_mean[i-1]) > limit:
            res[i-1] = 1
            res[i] = 1
            if i < (len(r_mean)-1):
                res[i+1] = 1
        
    return res
def compute_rolling(arr, kernel_a, kernel_b,limit):
    start = kernel_a + kernel_b
    pred = np.zeros(len(arr))
    pred_std = np.zeros(len(arr))
    pred_normal = np.zeros(len(arr))
    
    
    for i in range(start,len(arr)):
        a_sset = arr[(i-start):(i-kernel_b)]
        b_sset = arr[(i-kernel_b):i]
        
        a_mu = np.mean(a_sset)
        a_std = np.std(a_sset)
        
        b_mu = np.mean(b_sset)
        b_std = np.std(b_sset)
        
#         print(a_sset,b_sset,arr[i])
        
        
#         if abs(b_mu-a_mu) >= limit * abs(a_std):
#             pred[i] = 1
            
        if abs(b_std-a_std) >= limit * abs(a_std):
            pred_std[i-1] = 1
        
#         if arr[i] > (b_mu + limit * b_std) or arr[i] < (b_mu - limit * b_std):
#             pred_normal[i] = 1
            
            
    return pred, pred_std, pred_normal    
z = [i for i in range(20)]
compute_rolling(z,5,5,0.5)<jupyter_output>[0, 1, 2, 3, 4] [5, 6, 7, 8, 9] 10
[1, 2, 3, 4, 5] [6, 7, 8, 9, 10] 11
[2, 3, 4, 5, 6] [7, 8, 9, 10, 11] 12
[3, 4, 5, 6, 7] [8, 9, 10, 11, 12] 13
[4, 5, 6, 7, 8] [9, 10, 11, 12, 13] 14
[5, 6, 7, 8, 9] [10, 11, 12, 13, 14] 15
[6, 7, 8, 9, 10] [11, 12, 13, 14, 15] 16
[7, 8, 9, 10, 11] [12, 13, 14, 15, 16] 17
[8, 9, 10, 11, 12] [13, 14, 15, 16, 17] 18
[9, 10, 11, 12, 13] [14, 15, 16, 17, 18] 19<jupyter_text>IMPLEMENTATION<jupyter_code>input_dir = './../../train/KPI/'

# summary = pd.DataFrame(columns=['KPI', 'TP', 'TN', 'FP', 'FN', 'PRECISION', 'RECALL', 'F1_SCORE'])
summary_2 = pd.DataFrame(columns=['KPI', 'TP', 'TN', 'FP', 'FN', 'PRECISION', 'RECALL', 'F1_SCORE'])
# summary_3 = pd.DataFrame(columns=['KPI', 'TP', 'TN', 'FP', 'FN', 'PRECISION', 'RECALL', 'F1_SCORE'])


for fname in os.listdir(input_dir):
    limit = 1
    
    high_1 = 0
    high_2 = 0
    high_3 = 0
    
    while limit <= 12:
    
        df  = pd.read_csv(os.path.join(input_dir, fname), index_col='timestamp')
        kpi_name = df['KPI ID'].values[0]
        print(kpi_name)
        df = df.drop(['KPI ID'], axis=1)


        # Make Predictions
        y_pred, y_std_pred, y_normal_pred = compute_rolling(df.value.values,5,5,limit)

        # Generate Confusion Matrix
#         cf = conf_mat(y_pred,df.label.values)
        cf_2 = conf_mat(y_std_pred,df.label.values)
#         cf_3 = conf_mat(y_normal_pred,df.label.values)


        # F1-Score
#         f1, prec, rec = f1_comp(cf)
        f1_2, prec_2, rec_2 = f1_comp(cf_2)
#         f1_3, prec_3, rec_3 = f1_comp(cf_3)


#         if f1 > high_1:
#             res_1 = [ cf[0], cf[1], cf[2], cf[3], prec, rec, f1,limit ]
#             high_1 = f1
            
        if f1_2 > high_2:
            res_2 = [ cf[0], cf[1], cf[2], cf[3], prec_2, rec_2, f1_2,limit ]
            high_2 = f1_2
            
#         if f1_3 > high_3:
#             res_3 = [ cf[0], cf[1], cf[2], cf[3], prec_3, rec_3, f1_3,limit ]
#             high_3 = f1_3
        
        limit += 0.1
        
        
#     summary = summary.append({'KPI': kpi_name, 
#                               'LIMIT': res_1[7],
#                                'TP': res_1[0],  
#                                'TN': res_1[1], 
#                                'FP': res_1[2], 
#                                'FN': res_1[3], 
#                                'PRECISION': res_1[4], 
#                                'RECALL': res_1[5],     
#                                'F1_SCORE': res_1[6] }, ignore_index=True)

    summary_2 = summary_2.append({'KPI': kpi_name, 
                                  'LIMIT': res_2[7],
                                   'TP': res_2[0],  
                                   'TN': res_2[1], 
                                   'FP': res_2[2], 
                                   'FN': res_2[3], 
                                   'PRECISION': res_2[4], 
                                   'RECALL': res_2[5],     
                                   'F1_SCORE': res_2[6] }, ignore_index=True)

#     summary_3 = summary_3.append({'KPI': kpi_name, 
#                                   'LIMIT': res_3[7],
#                                    'TP': res_3[0],  
#                                    'TN': res_3[1], 
#                                    'FP': res_3[2], 
#                                    'FN': res_3[3], 
#                                    'PRECISION': res_3[4], 
#                                    'RECALL': res_3[5],     
#                                    'F1_SCORE': res_3[6] }, ignore_index=True)

    
    
summary_2.to_csv('RollStd_A_B.csv')
summary.to_csv('Rolling_STD_1.csv')

summary_2.to_csv('Rolling_STD_2.csv')

summary_3.to_csv('Rolling_STD_3.csv')
summary
summary_2
summary_2.to_csv('Rolling_Mean_5_set.csv')

summary_2.to_csv('Rolling_Mean_5.csv')<jupyter_output><empty_output><jupyter_text>Single Model<jupyter_code>input_dir = './../../train/KPI/'
fname = 'train_046ec29ddf80d62e.csv'

test = pd.DataFrame(columns=['KPI', 'TP', 'TN', 'FP', 'FN', 'PRECISION', 'RECALL', 'F1_SCORE'])

limit = 1.0

while limit < 18:


    df  = pd.read_csv(os.path.join(input_dir, fname), index_col='timestamp')
    kpi_name = df['KPI ID'].values[0]
    print(kpi_name)
    df = df.drop(['KPI ID'], axis=1)


    # Make Predictions
    y_pred, y_std_pred, y_normal_pred = compute_rolling(df.value.values,5,5,limit)

    # Generate Confusion Matrix
    cf = conf_mat(y_pred,df.label.values)
    cf_2 = conf_mat(y_normal_pred,df.label.values)
    cf_3 = conf_mat(y_std_pred,df.label.values)
    

    # F1-Score
    f1, prec, rec = f1_comp(cf)
    f1_2, prec_2, rec_2 = f1_comp(cf_2)
    f1_3, prec_3, rec_3 = f1_comp(cf_3)
    

    print(limit,f1,f1_2,f1_3)

    test = test.append({'KPI': kpi_name,
                              'STD': limit,
                                   'TP': cf[0],  
                                   'TN': cf[1], 
                                   'FP': cf[2], 
                                   'FN': cf[3], 
                                   'PRECISION': prec, 
                                   'RECALL': rec,     
                                   'F1_SCORE': f1 }, ignore_index=True)
    
    limit += 0.1
    limit = round(limit,1)
y_normal_pred
test[test.index == 91]['F1_SCORE']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15603224484219155, 'fraction_numerical': 0.056155212460718675, 'mean_word_length': 1.4605042016806722, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '409058', 'license': 'MIT', 'n_tokens_mistral': 3065, 'n_tokens_neox': 2796, 'n_words': 552, 'path': 'THU--Advanced_Network_Management/GP/Models/RollMean/Std.ipynb', 'repo_name': 'AlbertMillan/THU--ACM_2019-2021', 'size': '7319'}