<jupyter_start><jupyter_text>Balance training data> Various methods to balance training data by target class (label) for model training<jupyter_code># hide
from nbdev.showdoc import *
from fastcore.test import *
# export 
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Downsample<jupyter_code># export
def downsample(df:pd.DataFrame, y_column:str, min_size:int=None, random_state:int=115, **kwargs) -> pd.DataFrame:
    '''Balance classes of the target variable by downsampling all classes to be equal to or smaller than "min_size".
    
    Classes smaller than "min_size" are not affected and will remain at their current size. If "min_size" is ommitted, 
    the size of the smallest current class is taken as "min_size".   
    '''
    df_new = df.copy()
    
    # get smallest current class if not supplied
    if min_size == None:
        min_size = df_new[y_column].value_counts().min()
    
    # downsample all classes larger than min_size
    for class_index, group in df_new.groupby(y_column):
        if group.shape[0] > min_size:
            drop_idx = group.sample(len(group)-min_size, random_state=random_state, **kwargs).index
            df_new = df_new.drop(drop_idx)
            
    return df_new.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>**Paramters**- *df*: Dataframe containing column "y_column"- *y_column*: Name of df column containing the target variable (label)- *min_size*: If no value is supplied, min_size will be set to the size of the smallest current class- *random_state*: Random state for reproducibility**Returns**- *new_df*: Has the same structure as the input dataframe but classes were balanced by downsampling<jupyter_code>df = pd.DataFrame({"x": ["A1", "B1", "B2", "C1", "C2", "C3"], "y": [0, 1, 1, 2, 2, 2]})
df.groupby("y").count()
new_df = downsample(df=df, y_column="y", random_state=115)
new_df.groupby("y").count()
# hide

# downsample unit tests
# test 0 
new_df = downsample(df=df, y_column="y", min_size=0).groupby("y").count()
test_eq(list(new_df.x), [])

# test default downsample 
new_df = downsample(df=df, y_column="y").groupby("y").count()
test_eq(list(new_df.x), [1, 1, 1])

# test partial downsample
new_df = downsample(df=df, y_column="y", min_size=2).groupby("y").count()
test_eq(list(new_df.x), [1, 2, 2])

# test none downsample
new_df = downsample(df=df, y_column="y", min_size=3).groupby("y").count()
test_eq(list(new_df.x), [1, 2, 3])

# test too large
new_df = downsample(df=df, y_column="y", min_size=4).groupby("y").count()
test_eq(list(new_df.x), [1, 2, 3])<jupyter_output><empty_output><jupyter_text>Upsample<jupyter_code># export

def upsample(df:pd.DataFrame, y_column:str, max_size:int=None, random_state:int=115, **kwargs) -> pd.DataFrame:
    '''Balance classes of the target variable by upsampling all classes to be equal to or larger than "max_size".
       
    Classes larger than "max_size" are not affected and will remain at their current size. If "max_size" is ommitted, 
    the size of the largest class is taken as "max_size".   
    '''
    df_new = df.copy()
    
    # get largest current class if not supplied
    if max_size == None:
        max_size = df_new[y_column].value_counts().max()
    
    # upsample all classes smaller than max_size
    for class_index, group in df_new.groupby(y_column):
        if group.shape[0] < max_size:
            df_new = df_new.append(group.sample(max_size-len(group), replace=True, random_state=random_state, **kwargs))
        
    return df_new.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>**Paramters**- *df*: Dataframe containing column "y_column"- *y_column*: Name of df column containing the target variable (label)- *max_size*: If no value is supplied, max_size will be set to the size of the largest current class- *random_state*: Random state for reproducibility**Returns**- *new_df*: Has the same structure as the input dataframe but classes were balanced by upsampling<jupyter_code>df = pd.DataFrame({"x": ["A1", "B1", "B2", "C1", "C2", "C3"], "y": [0, 1, 1, 2, 2, 2]})
df.groupby("y").count()
new_df = upsample(df=df, y_column="y", random_state=115)
new_df.groupby("y").count()
# hide

# upsample unit tests
# test 0 
new_df = upsample(df=df, y_column="y", max_size=0).groupby("y").count()
test_eq(list(new_df.x), [1, 2, 3])

# test default upsample 
new_df = upsample(df=df, y_column="y").groupby("y").count()
test_eq(list(new_df.x), [3, 3, 3])

# test partial upsample
new_df = upsample(df=df, y_column="y", max_size=2).groupby("y").count()
test_eq(list(new_df.x), [2, 2, 3])

# test explicit default upsample
new_df = upsample(df=df, y_column="y", max_size=3).groupby("y").count()
test_eq(list(new_df.x), [3, 3, 3])

# test complete upsample
new_df = upsample(df=df, y_column="y", max_size=4).groupby("y").count()
test_eq(list(new_df.x), [4, 4, 4])<jupyter_output><empty_output><jupyter_text>Balance<jupyter_code># export

def balance(df:pd.DataFrame, y_column:str, size:int, random_state:int=115, **kwargs) -> pd.DataFrame:
    '''Balance classes of the target variable by up- or downsampling all classes to be equal to "size".
    '''
    df_new = df.copy()
    
    for class_index, group in df_new.groupby(y_column):
        if group.shape[0] < size:
            df_new = df_new.append(group.sample(size-len(group), replace=True, random_state=random_state, **kwargs))
        if group.shape[0] > size:
            drop_idx = group.sample(len(group)-size, random_state=random_state, **kwargs).index
            df_new = df_new.drop(drop_idx)
        
    return df_new.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>**Paramters**- *df*: Dataframe containing column "y_column"- *y_column*: Name of df column containing the target variable (label)- *size*: desired class size- *random_state*: Random state for reproducibility**Returns**- *new_df*: Has the same structure as the input dataframe but classes were balanced to a certain size<jupyter_code>df = pd.DataFrame({"x": ["A1", "B1", "B2", "C1", "C2", "C3"], "y": [0, 1, 1, 2, 2, 2]})
df.groupby("y").count()
new_df = balance(df=df, y_column="y", random_state=115, size=5)
new_df.groupby("y").count()
# hide

# balance unit tests
# test 0 
new_df = balance(df=df, y_column="y", size=0).groupby("y").count()
test_eq(list(new_df.x), [])

# test partial balance
new_df = balance(df=df, y_column="y", size=1).groupby("y").count()
test_eq(list(new_df.x), [1, 1, 1])

# test partial balance
new_df = balance(df=df, y_column="y", size=3).groupby("y").count()
test_eq(list(new_df.x), [3, 3, 3])

# test large balance
new_df = balance(df=df, y_column="y", size=4).groupby("y").count()
test_eq(list(new_df.x), [4, 4, 4])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15220293724966621, 'fraction_numerical': 0.01631805370123127, 'mean_word_length': 4.914035087719299, 'pattern_counts': {'":': 6, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '317592', 'license': 'Apache-2.0', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2406, 'n_words': 627, 'path': '00_training.balance.ipynb', 'repo_name': 'maronr/ds_helpers', 'size': '6741'}