<jupyter_start><jupyter_text>Split into Train and Test SetsFor Obstacle and Surface Problem classes, do 50/50For Curb Ramp Present, aim for 80/20<jupyter_code>import pandas as pd
import os
import s3fs # for reading from S3FileSystem
import json # for working with JSON files 

import matplotlib.pyplot as plt

pd.set_option('max_colwidth', -1)<jupyter_output><empty_output><jupyter_text>Load in Labels with Metadata<jupyter_code>df = pd.read_csv('single_labels_with_metadata.csv')
print(df.shape)
df.head()
df['label_name'].value_counts()<jupyter_output><empty_output><jupyter_text>Work to split on Obstructions<jupyter_code>df_ob = df.loc[df['label_name'].str.contains('Obstacle')]
df_ob.shape
# Get number of panos
obstruction_img_ids = list(df_ob['img_id'].unique())
len(obstruction_img_ids)<jupyter_output><empty_output><jupyter_text>Split on Surface Problem<jupyter_code>df_sp = df.loc[df['label_name'].str.contains('Surface')]
df_sp.shape
# Get number of panos
surface_prob_img_ids = list(df_sp['img_id'].unique())
len(surface_prob_img_ids)<jupyter_output><empty_output><jupyter_text>Split on Surface Problem<jupyter_code>df_miss = df.loc[df['label_name'].str.contains('Missing')]
df_miss.shape
# Get number of pano
missing_img_ids = list(df_miss['img_id'].unique())
len(missing_img_ids)<jupyter_output><empty_output><jupyter_text>Try to get a groupby img_id<jupyter_code>bool_cols = ['present_ramp', 'missing_ramp', 'obstacle', 'surface_prob', 'no_sidewalk']
count_cols = ['labels_count', 'count_present_ramp', 'count_missing_ramp', 'count_obstacle', 'count_surface_prob', 'count_no_sidewalk']
df_img_id = df.groupby(['img_id', 'pano_id'])[bool_cols].any()
df_img_id.head()
df_img_id.shape
df_img_id['labels_count'] = df.groupby(['img_id', 'pano_id'])['region_count'].count()
df_img_id.head()
df_img_id_counts = df.groupby(['img_id', 'pano_id'])[bool_cols].sum()
df_img_id_counts = df_img_id_counts.add_prefix('count_')
df_img_id_counts.head()
df_img = pd.merge(df_img_id, df_img_id_counts, left_index = True, right_index = True)
df_img.head()
df_img = df_img.reset_index()
df_img.head()<jupyter_output><empty_output><jupyter_text>Attempt and Evaluate some Splits!<jupyter_code>rare_mask = (df_img['missing_ramp'] | df_img['obstacle'] | df_img['surface_prob'])
df_img_rare = df_img.loc[rare_mask]
print(df_img_rare.shape)
df_img_rare.head()
df_img_rare = df_img_rare.reset_index()
# Save all rare images 
# This does not include any split information 
df_img_rare.to_csv('data-split/all_rare_panos.csv', index = False)
df_img_rare[bool_cols].sum()
df_img_rare[count_cols].sum()
# Work on splitting with a random number generator 
# Try 40% 
df_img_rare_test = df_img_rare.sample(frac = 0.4)
print(df_img_rare_test.shape)
df_img_rare_test[bool_cols].sum()
df_img_rare_test[count_cols].sum()
df_img_rare_test = df_img_rare_test.reset_index()
df_img_rare_test.head()
#df_img_rare_test.to_csv('data-split/rare_test_set_60-40.csv')
# Try to add train_test bool
test_rare_pano_id = list(df_img_rare_test['pano_id'])
is_test_pano = df_img_rare['pano_id'].isin(test_rare_pano_id)
is_test_pano.head()
is_test_pano.value_counts()
df_img_rare['is_test_pano'] = is_test_pano
df_img_rare_train = df_img_rare.loc[~df_img_rare['is_test_pano']]
print(df_img_rare_train.shape)
print(df_img_rare_train[count_cols].sum())
print('\n')
print(df_img_rare_train[bool_cols].sum())
# Save rare
df_img_rare_train.to_csv('data-split/rare_train_only_6040.csv', index = False)
df_img_rare.head()
# Save rare
df_img_rare.to_csv('data-split/all_rare_with_is_test_4060.csv', index = False)
df_img_rare['is_test_pano'].value_counts()
df_img_rare['includes_rare'] = True<jupyter_output><empty_output><jupyter_text>Select from CommonAim for 51 out of 259 from the common group, so about 20%<jupyter_code>type(df_img)
df_img.shape
type(rare_mask)
rare_mask.shape
df_img_common = df_img.loc[~rare_mask.values]
print(df_img_common.shape)

print(df_img_common[bool_cols].sum())
print('\n')
print(df_img_common[count_cols].sum())

df_img_common.head()
print(df_img_common[bool_cols].sum() * 0.2)
print('\n')
print(df_img_common[count_cols].sum() * 0.2)
df_img_common_test = df_img_common.sample(n = 51)
print(df_img_common_test.shape)
print(df_img_common_test[bool_cols].sum())
print('\n')
print(df_img_common_test[count_cols].sum())
# Save the test image
df_img_common_test.head()
df_img_common_test.to_csv(os.path.join('data-split', 'common_label_test_8020.csv'), index = False)
test_common_pano_id = list(df_img_common_test['pano_id'])
is_common_test_pano = df_img_common['pano_id'].isin(test_common_pano_id)
is_common_test_pano.head()
is_common_test_pano.value_counts()
df_img_common['is_test_pano'] = is_common_test_pano
df_img_common['includes_rare'] = False
df_img_common.head()
df_img_common.to_csv(os.path.join('data-split', 'all_common_images_8020.csv'), index = False)
df_img_common_train = df_img_common.loc[~df_img_common['is_test_pano']]
df_img_common_train.to_csv(os.path.join('data-split', 'common_images_train_8020.csv'), index = False)<jupyter_output><empty_output><jupyter_text>Combine into final dataset<jupyter_code>df_all = pd.concat([df_img_common, df_img_rare])
print(df_all.shape)
print(df_all['is_test_pano'].value_counts())
df_all.head(10)
df_all.tail()
# Cast count columns to integers
for col in count_cols:
    df_all[col] = pd.to_numeric(df_all[col], downcast = 'integer')
df_all.head()
df_all.tail()
print(df_all[bool_cols].sum())
print('\n')
print(df_all[count_cols].sum())
df_all.to_csv(os.path.join('data-split', 'final_combined_splits.csv'), index = False)<jupyter_output><empty_output><jupyter_text>Generate a Summary Table<jupyter_code>total_pano_count = df_all[bool_cols].sum()
total_label_count = df_all[count_cols].sum()
test_pano_count = df_all[bool_cols].loc[df_all['is_test_pano']].sum()
test_label_count = df_all[count_cols].loc[df_all['is_test_pano']].sum()
train_pano_count = df_all[bool_cols].loc[~df_all['is_test_pano']].sum()
train_label_count = df_all[count_cols].loc[~df_all['is_test_pano']].sum()
print(total_pano_count)
print(total_label_count)
print(train_pano_count)
print(train_label_count)
print(test_pano_count)
print(test_label_count)

df_all_groupby = df_all.groupby(['is_test_pano'])[[*bool_cols, *count_cols]].sum()
df_all_groupby
combined_cols = [*bool_cols, *count_cols]
total_sum = df_all_groupby.sum()
total_sum
df_all_groupby = df_all_groupby.append(total_sum, ignore_index = True)
df_all_groupby['grouping'] = pd.Series(['Train', 'Test', 'Combined'])
df_all_groupby.head()
df_all_groupby_index = df_all_groupby.set_index('grouping')
df_all_groupby_index.head()
df_all_groupby_transpose = df_all_groupby_index.transpose()
#df_all_groupby_transpose = pd.to_numeric(df_all_groupby_transpose, downcast = 'integer')
df_all_groupby_transpose.to_csv(os.path.join('data-split', 'summary_split_transpose.csv'), float_format = '%.0f')
df_all_groupby_transpose
df_all_groupby_unstack = df_all_groupby_index.unstack()
df_all_groupby_unstack = pd.to_numeric(df_all_groupby_unstack, downcast = 'integer')
df_all_groupby_unstack
df_all_groupby_unstack.to_csv(os.path.join('data-split', 'summary_split_tall.csv'), header = False)
df_all_groupby_unstack.pivot(index = None, columns = 'grouping')
#(index='foo', columns='bar', values='baz')
df_all_groupby_pivot.index<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1372522214627478, 'fraction_numerical': 0.007518796992481203, 'mean_word_length': 10.186544342507645, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '479041', 'license': 'MIT', 'n_tokens_mistral': 3033, 'n_tokens_neox': 2986, 'n_words': 382, 'path': 'split-train-test/2020-03-22-SplitLabels.ipynb', 'repo_name': 'w210-accessibility/classify-streetview', 'size': '7315'}