<jupyter_start><jupyter_text>Test Sets<jupyter_code>ENG_TEST_SET = pd.read_csv(os.path.join(args.loc,'test','trac2_eng_test.csv'))
HIN_TEST_SET = pd.read_csv(os.path.join(args.loc,'test','trac2_hin_test.csv'))
IBEN_TEST_SET = pd.read_csv(os.path.join(args.loc,'test','trac2_iben_test.csv'))<jupyter_output><empty_output><jupyter_text>Helper Functions<jupyter_code>def get_analysis(
    gold_csv_loc:str,
    pred_csv_loc:str,
    test_set_df:pd.DataFrame,
    subtask:str
):
    """
    Args:
        gold_csv_loc: Location of gold label csv file
        pred_csv_loc: Location of csv with predicted labels
        test_set_df: Df containing the 'ID' and corresponding text
        subtask: either 'a' or 'b'
    Returns:
        analysis_df: DF containing the columns  Text, True labels as 'true'
            predicted labels as 'pred'
        cross_tab: A pandas crosstab as confusion matrix
    """
    gold_df = pd.read_csv(gold_csv_loc)
    preds_df = pd.read_csv(pred_csv_loc)
    preds_df.rename(
        columns={'Label':'pred'},
        inplace=True,
    )
    temp = pd.merge(
        test_set_df,
        gold_df, on='ID'
    )
    temp.rename(
        columns={f'Sub-task {subtask.upper()}': 'true'},
        inplace=True
    )
    analysis_df = pd.merge(temp, preds_df, on='ID')
    cross_tab = pd.crosstab(
        analysis_df.true , #y_true
        analysis_df.pred , #y_pred
        rownames=['True'], colnames=['Predicted'], margins=True
    )
    return analysis_df, cross_tab
def print_samples(
    analysis_df:pd.DataFrame,
    mistakes:bool=True,
    num_samples:int=5
)-> None:
    """Prints the samples for analysis"""
    
    with pd.option_context('display.max_colwidth', -1): 
        if mistakes: #print misclassifications
            df = analysis_df[analysis_df.true != analysis_df.pred]
            [['Text','true','pred']]
        else: #print correct classifications
            df = analysis_df[analysis_df.true == analysis_df.pred]
            [['Text','true','pred']]
            
        print(df.sample(num_samples))
        
        print('\n',df['Text'].map(len).describe())<jupyter_output><empty_output><jupyter_text>Eng Sub-Task A Analysis<jupyter_code>eng_a_analysis_df, eng_a_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_eng_gold_a.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','task_a_eng_trac_labels_kaushik.csv'),
    test_set_df = ENG_TEST_SET,
    subtask = 'a',
)

print(classification_report(
    y_true = eng_a_analysis_df.true,
    y_pred = eng_a_analysis_df.pred,
    digits=4,
))
eng_a_analysis_df
eng_a_analysis_df[eng_a_analysis_df.Text=="as long as women vote, they won't change this present situation"]
_t = pd.read_csv(os.path.join(args.loc,'gold','trac2_eng_gold_a.csv'))
_t[_t.ID=='C24.36']<jupyter_output><empty_output><jupyter_text>Correct Predictions<jupyter_code>print_samples(
    analysis_df = eng_a_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID  \
914   C60.694   
760   C8.1386   
1120  C18.84    
121   C52.106   
976   C60.966   

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 [...]<jupyter_text>Incorrect Predictions<jupyter_code>print_samples(
    analysis_df = eng_a_analysis_df,
    mistakes = True,
    num_samples = 5
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        eng_a_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/eng_a_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Eng Sub-Task B Analysis<jupyter_code>eng_b_analysis_df, eng_b_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_eng_gold_b.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','trac_task_b_eng_preds_kaushik.csv'),
    test_set_df = ENG_TEST_SET,
    subtask = 'b',
)
    

print(classification_report(
    y_true = eng_b_analysis_df.true,
    y_pred = eng_b_analysis_df.pred,
    digits=4,
))
eng_b_analysis_df<jupyter_output><empty_output><jupyter_text>Correct Predictions<jupyter_code>print_samples(
    analysis_df = eng_b_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID  \
37    C37.15      
471   C24.786.5   
356   C24.222     
1064  C18.43      
620   C24.1691    

                                                                                                                                                      Text  \
37    interrupting the pulitzer prize winning author? you dumbo burkha                                                                                       
471   @sid chawla  \nfeminism =slut empowerment phase 2  \ntraditionalism =slut empowerment phase 1                                                          
356   2.1k people, who dislike it, are feminists 😊                                                                                                           
1064  such a fine actor by his own instinct 👏👏👏👏                                                                                                             
620   i think only 5% women /girls miss use this think.  \nmostly i have seen india's sons faul[...]<jupyter_text>Incorrect Predictions<jupyter_code>print_samples(
    analysis_df = eng_b_analysis_df,
    mistakes = True,
    num_samples =5
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        eng_b_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/eng_b_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Hindi Sub-Task A Analysis<jupyter_code>hin_a_analysis_df, hin_a_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_hin_gold_a.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','hindi_task_a_preds.csv'),
    test_set_df = HIN_TEST_SET,
    subtask = 'a',
)
    

print(classification_report(
    y_true = hin_a_analysis_df.true,
    y_pred = hin_a_analysis_df.pred,
    digits=4,
))<jupyter_output>precision    recall  f1-score   support

         CAG     0.5025    0.5288    0.5153       191
         NAG     0.7159    0.7908    0.7515       325
         OAG     0.9203    0.8611    0.8897       684

    accuracy                         0.7892      1200
   macro avg     0.7129    0.7269    0.7188      1200
weighted avg     0.7984    0.7892    0.7927      1200<jupyter_text>Correctly classified<jupyter_code>print_samples(
    analysis_df = hin_a_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID  \
583   C24.3674   
586   C24.3744   
211   C24.978    
935   C8.2547    
1099  C8.4148    

                                                                                                            Text  \
583   377 dislikes wale randi k gand mai garam loha                                                                
586   feminism ka galat upyog ho raha hai                                                                          
211   यहाँ तो साबित होने से पहले न्यूज़ पेपर में लड़के का नाम आ जाता है                                              
935   bhag teri maa ki chut..bhag ja yt se..yaha mat haag..jute jaise mu wali..yt pe\nhaag k gandagi mat faila..   
1099  didi tumhari maa ka bhosda :)                                                                                

     true pred  
583   OAG  OAG  
586   CAG  CAG  
211   NAG  NAG  
935   OAG  OAG  
1099  OAG  OAG  

 count    947.000000 
mean     117.600845 
std      186.663316 
min      2.000000   
25%   [...]<jupyter_text>Incorrectly classified<jupyter_code>print_samples(
    analysis_df = hin_a_analysis_df,
    mistakes = True,
    num_samples =5
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        hin_a_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/hin_a_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Hindi Sub-Task B Analysis<jupyter_code>hin_b_analysis_df, hin_b_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_hin_gold_b.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','hindi_task_b_preds.csv'),
    test_set_df = HIN_TEST_SET,
    subtask = 'b',
)
    

print(classification_report(
    y_true = hin_b_analysis_df.true,
    y_pred = hin_b_analysis_df.pred,
    digits=4,
))<jupyter_output>precision    recall  f1-score   support

         GEN     0.8825    0.8342    0.8577       567
        NGEN     0.8584    0.9005    0.8790       633

    accuracy                         0.8692      1200
   macro avg     0.8704    0.8673    0.8683      1200
weighted avg     0.8698    0.8692    0.8689      1200<jupyter_text>Correctly classified<jupyter_code>print_samples(
    analysis_df = hin_b_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID  \
946   C8.2681     
1166  C8.4649     
19    C60.543     
1138  C8.4396     
135   C24.531.3   

                                                                                         Text  \
946   fucking fake feminist  \nnikal lav**i pehli fursat me nikal koi jarurat nahi teri idhar   
1166  teri chut chodni hai babu                                                                 
19    nice video  \njio group  \npati dil rani didi ke                                          
1138  madarchod gand marbha aur apne bhap sa pouh ki kya galti kar de                           
135   @dhananjay r hunasgi arre bole toh haan...didnt u read my reply                           

      true  pred  
946   GEN   GEN   
1166  GEN   GEN   
19    NGEN  NGEN  
1138  GEN   GEN   
135   NGEN  NGEN  

 count    1043.000000
mean     116.627996 
std      170.236128 
min      2.000000   
25%      35.000000  
50%      66.000000  
75%      134.500000 
max      2907.000000
Name: Text, dtype:[...]<jupyter_text>Incorrectly classified<jupyter_code>print_samples(
    analysis_df = hin_b_analysis_df,
    mistakes = True,
    num_samples =5
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        hin_b_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/hin_b_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Bengali Sub-Task A Analysis<jupyter_code>iben_a_analysis_df, iben_a_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_iben_gold_a.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','iben_task_a_preds.csv'),
    test_set_df = IBEN_TEST_SET,
    subtask = 'a',

print(classification_report(
    y_true = iben_a_analysis_df.true,
    y_pred = iben_a_analysis_df.pred,
    digits=4,
))<jupyter_output>precision    recall  f1-score   support

         CAG     0.6053    0.5111    0.5542       225
         NAG     0.8446    0.9087    0.8755       712
         OAG     0.7802    0.7211    0.7495       251

    accuracy                         0.7938      1188
   macro avg     0.7434    0.7136    0.7264      1188
weighted avg     0.7857    0.7938    0.7880      1188<jupyter_text>Correctly classified<jupyter_code>print_samples(
    analysis_df = iben_a_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID  \
369   C60.308    
615   C8.12276   
1163  C60.1011   
180   C60.54     
998   C60.782    

                                                                     Text  \
369   dada er por tomar gaan viral hbe 🤭🤭                                   
615   randi                                                                 
1163  😁😁ata last te video banale ata to kat agei bananor katha boss😁😁😍😍😍😍   
180   btw tomar hasi ta khub sundor 🤓                                       
998   0 tka hiro hoa atto poriborton  \nsotti ucit na.                      

     true pred  
369   NAG  NAG  
615   OAG  OAG  
1163  NAG  NAG  
180   NAG  NAG  
998   CAG  CAG  

 count    943.000000
mean     36.625663 
std      45.776491 
min      1.000000  
25%      13.000000 
50%      23.000000 
75%      44.000000 
max      625.000000
Name: Text, dtype: float64<jupyter_text>Incorrectly classified<jupyter_code>print_samples(
    analysis_df = iben_a_analysis_df,
    mistakes = True,
    num_samples =5
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        iben_a_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/iben_a_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Bengali Sub-Task B Analysis<jupyter_code>iben_b_analysis_df, iben_b_crosstab = get_analysis(
    gold_csv_loc = os.path.join(args.loc,'gold','trac2_iben_gold_b.csv'),
    pred_csv_loc = os.path.join(args.loc,'runs','iben_task_b_preds.csv'),
    test_set_df = IBEN_TEST_SET,
    subtask = 'b',
)
print(classification_report(
    y_true = iben_b_analysis_df.true,
    y_pred = iben_b_analysis_df.pred,
    digits=4,
))<jupyter_output>precision    recall  f1-score   support

         GEN     0.7315    0.7822    0.7560       202
        NGEN     0.9547    0.9412    0.9479       986

    accuracy                         0.9141      1188
   macro avg     0.8431    0.8617    0.8519      1188
weighted avg     0.9168    0.9141    0.9153      1188<jupyter_text>Correctly classified<jupyter_code>print_samples(
    analysis_df = iben_b_analysis_df,
    mistakes = False,
    num_samples =5
)<jupyter_output>ID                                                 Text  true  pred
1131  C60.967  last kotha gulo darun dada 🤗🤗..  \nvideo ta darun☺☺  NGEN  NGEN
1064  C60.874  darun hoyeche                                        NGEN  NGEN
979   C60.757  da bhai carry on 👍👍                                  NGEN  NGEN
272   C60.176  ranu pagli chilo r pagli tahkbe                      NGEN  NGEN
732   C16.185  sasti copy...pagli chudi                             GEN   GEN 

 count    1086.000000
mean     41.186004  
std      52.744357  
min      1.000000   
25%      14.000000  
50%      25.000000  
75%      49.000000  
max      625.000000 
Name: Text, dtype: float64<jupyter_text>Incorrectly classified<jupyter_code>print_samples(
    analysis_df = iben_b_analysis_df,
    mistakes = True,
    num_samples = 5,  
)
with sns.plotting_context('notebook',font_scale=1.6):
    f, ax = plt.subplots(figsize=(6, 6))
    ax = sns.heatmap(
        iben_b_crosstab,
        annot=True,
        fmt="d", 
        cmap= 'Blues',
        annot_kws={"size": 20},
    )
    f.savefig(
        'confmats/iben_b_confmat.eps', format='eps', dpi=600
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08816105010728259, 'fraction_numerical': 0.06866086078505616, 'mean_word_length': 1.608559670781893, 'pattern_counts': {'":': 6, '<': 71, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 103, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '331882', 'license': 'MIT', 'n_tokens_mistral': 6044, 'n_tokens_neox': 5364, 'n_words': 1035, 'path': 'notebooks/TRAC Error Analysis.ipynb', 'repo_name': 'cozek/trac2020_submission', 'size': '15846'}