<jupyter_start><jupyter_text>Preprocessing - Data LabelingLoad in the csv files prepared in notebook 1.1 and create text labels for multilabel classificaiton. Experiments- Upsampling - Explore whether further upsampling of the minority class improves model performance. This will be run in addition to the original dataset.- Including Negative Observations - in notebook 1.1 I included the negative or null observations that were included in the tiles.zip file and were used in the original study. I'll explore how using these observations changes the model performance.<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline

from fastai.vision import *
from fastai.metrics import error_rate
import os 

import config as cfg<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>current_module = sys.modules[__name__]
csvs = ['tiles_train', 'tiles_validation', 'tiles_test']
for csv in csvs:
    print('loading {}'.format(csv))
    df_name = csv
    setattr(current_module,
            df_name,
            pd.read_csv(os.path.join(cfg.csv_dir, '{}.csv'.format(csv))))
    df = getattr(current_module, df_name)
    print(df.shape)
    display(df.head(2))
train = tiles_train[['imagename'] + cfg.image_classes].copy()
validation = tiles_validation[['imagename'] + cfg.image_classes].copy()
test = tiles_test[['imagename'] + cfg.image_classes].copy()<jupyter_output><empty_output><jupyter_text>Create Label  The threshold used for labeling is the same as the original dataset. Lowering the threshold could create more observations.<jupyter_code>threshold = .99
for class_i in cfg.image_classes:
    train[class_i] = train[class_i].map(lambda x: class_i if x > threshold else '')
    validation[class_i] = validation[class_i].map(lambda x: class_i if x > threshold else '')
    test[class_i] = test[class_i].map(lambda x: class_i if x > threshold else '')
    
def create_text_label(df,
                       classes=[],
                       label_col_name='label',
                       negative_class_label='negative'):
    
    df[label_col_name] = df.apply(lambda row: ' '.join(filter(None,
                                                               [row[class_i] for class_i in classes] 
                                                              )), axis=1)
    
    # remove whitespace and label absence of any classes as negative class
    df[label_col_name] = df[label_col_name].map(lambda x: x.strip() if x.strip() else negative_class_label)
    return(df)


train = create_text_label(train,
                          classes=cfg.image_classes,
                          negative_class_label='negative')
validation = create_text_label(validation,
                               classes=cfg.image_classes,
                               negative_class_label='negative')
test = create_text_label(test,
                         classes=cfg.image_classes,
                         negative_class_label='negative')

train.drop(columns=cfg.image_classes, inplace=True)
validation.drop(columns=cfg.image_classes, inplace=True)
test.drop(columns=cfg.image_classes, inplace=True)
display(train.head(2))
display(validation.head(2))
display(test.head(2))<jupyter_output><empty_output><jupyter_text>Upsample core class<jupyter_code>core_upsample = train.loc[train.label.str.contains('cored')].sample(frac=0.4)
neg_upsample = train.loc[train.label.str.contains('negative')].sample(frac=1)
train_upsampled = pd.concat([train, core_upsample, neg_upsample])

assert (train.shape[0] + core_upsample.shape[0] + neg_upsample.shape[0]) == train_upsampled.shape[0]
from plotly.subplots import make_subplots
import plotly.graph_objects as go
import plotly.io as pio
pio.renderers.default = "iframe"
train_unique = train.drop_duplicates(subset=['imagename'], keep='first')
fig = make_subplots(rows=3, cols=1, shared_xaxes=True,
                   subplot_titles=("Train", "Validation", "Test"))
fig.append_trace(go.Histogram(x=train_unique['label'],
                           histnorm='percent',
                           name='train_unique'), row=1, col=1)
fig.add_trace(go.Histogram(x=train['label'],
                           histnorm='percent',
                           name='train_plaquebox-paper'), row=1, col=1)
fig.add_trace(go.Histogram(x=train_upsampled['label'],
                           histnorm='percent',
                           name='train_upsampled'), row=1, col=1)
fig.append_trace(go.Histogram(x=validation['label'],
                           histnorm='percent',
                           name='validation'), row=2, col=1)
fig.append_trace(go.Histogram(x=test['label'],
                           histnorm='percent',
                           name='test'), row=3, col=1)

fig.update_layout(title_text="Percent of labels in train, validation, and test sets")
fig.show()<jupyter_output><empty_output><jupyter_text>Format for FastaiAdd directory to file path in the DataFrame. Also, join train and validation for fastai data block api for index splitting in notebook 2.1.<jupyter_code>train['imagename'] = train['imagename'].map(lambda x: f"train/{x}")
train_upsampled['imagename'] = train_upsampled['imagename'].map(lambda x: f"train/{x}")
validation['imagename'] = validation['imagename'].map(lambda x: f"validation/{x}")
train_n_validation = pd.concat([train, validation])
train_n_validation.reset_index(inplace=True, drop=True)

assert train_n_validation.shape[0] == train.shape[0] + validation.shape[0]
train.to_csv(os.path.join(cfg.csv_dir, 'train_multilabel.csv'), index=False)
train_upsampled.to_csv(os.path.join(cfg.csv_dir, 'train_upsampled_multilabel.csv'), index=False)
validation.to_csv(os.path.join(cfg.csv_dir, 'validation_multilabel.csv'), index=False)
# train_n_validation.to_csv(os.path.join(cfg.csv_dir, 'train_n_validation_multilabel.csv'), index=False
test.to_csv(os.path.join(cfg.csv_dir, 'test_multilabel.csv'), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.117597435464822, 'fraction_numerical': 0.006073899105787076, 'mean_word_length': 3.588235294117647, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '464652', 'license': 'MIT', 'n_tokens_mistral': 1813, 'n_tokens_neox': 1784, 'n_words': 366, 'path': '1.2) Preprocessing - Data Labeling.ipynb', 'repo_name': 'DougForrest/plaquebox-paper', 'size': '5927'}