<jupyter_start><jupyter_text>English Train Set<jupyter_code>ENGLISH_TRAIN
with pd.option_context('display.max_colwidth', -1): 
    print(ENGLISH_TRAIN.sample(5))
ENGLISH_TRAIN.text.map(len).describe()
with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=3,sharey = True, figsize=(13, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B','Sub-Task C']

    _dfs = [
        ENGLISH_TRAIN['task_1'] ,
        ENGLISH_TRAIN['task_2'] ,  
        ENGLISH_TRAIN['task_3'] ,  
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("0.4"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+24))

    fig2.savefig(
        'images/english_train_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output><jupyter_text>English Test Set<jupyter_code>with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=3,sharey = True, figsize=(13, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B','Sub-Task C']

    _dfs = [
        ENGLISH_GOLD['task_1'] ,
        ENGLISH_GOLD['task_2'] ,  
        ENGLISH_GOLD['task_3'] ,  
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("b"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+24))

    fig2.savefig(
        'images/english_gold_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Hindi Train Set<jupyter_code>HINDI_TRAIN
with pd.option_context('display.max_colwidth', -1): 
    print(HINDI_TRAIN.sample(5))
HINDI_TRAIN.text.map(len).describe()
with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=3,sharey = True, figsize=(13, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B','Sub-Task C']

    _dfs = [
        HINDI_TRAIN['task_1'] ,
        HINDI_TRAIN['task_2'] ,  
        HINDI_TRAIN['task_3'] ,  
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("0.4"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+24))

    fig2.savefig(
        'images/hindi_train_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output><jupyter_text>Hindi Test Set<jupyter_code>with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=3,sharey = True, figsize=(13, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B','Sub-Task C']

    _dfs = [
        HINDI_GOLD['task_1'] ,
        HINDI_GOLD['task_2'] ,  
        HINDI_GOLD['task_3'] ,  
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("b"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+24))
                
    fig2.savefig(
        'images/hindi_gold_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output><jupyter_text>German Train Set<jupyter_code>GERMAN_TRAIN
with pd.option_context('display.max_colwidth', -1): 
    print(GERMAN_TRAIN.sample(5))
GERMAN_TRAIN.text.map(len).describe()
with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=2,sharey = True, figsize=(10, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B',]

    _dfs = [
        GERMAN_TRAIN['task_1'] ,
        GERMAN_TRAIN['task_2'] ,    
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("0.4"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+24))

    fig2.savefig(
        'images/german_train_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output><jupyter_text>German Test Set<jupyter_code>with sns.plotting_context('paper',font_scale=1.5):
    fig2, ax2 = plt.subplots(nrows=1, ncols=2,sharey = True, figsize=(10, 4))
    axes2 = ax2.flatten()
    fig2.tight_layout()

    subplot_titles = ['Sub-Task A','Sub-Task B']

    _dfs = [
        GERMAN_GOLD['task_1'] ,
        GERMAN_GOLD['task_2'] ,    
    ]
    for i,df in enumerate(_dfs):

        axp  = sns.countplot(df,ax = axes2[i], color=("b"))
        axes2[i].set_title(subplot_titles[i])
        axp.set_xlabel('label')
        
        for p in axp.patches:
                axp.annotate('{}'.format(p.get_height()), (p.get_x()+0.2, p.get_height()+5))

    fig2.savefig(
        'images/german_gold_countplot.pdf', format='pdf', dpi=600
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16862745098039217, 'fraction_numerical': 0.02969187675070028, 'mean_word_length': 3.069908814589666, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '256950', 'license': 'MIT', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2162, 'n_words': 279, 'path': 'notebooks/HASOC Data Exploration.ipynb', 'repo_name': 'cozek/hasoc_code', 'size': '5355'}