<jupyter_start><jupyter_text>Frame as a 2nd Stage of Binary Classification - Missing vs Present* The first model filters down to the "interesting" images - locations where we would expect there may be a cross walk* Then this model can focus on differentiating between missing, present, or not sure* Then humans can do some manual review of the images<jupyter_code>import pandas as pd
import numpy as np
import os
import boto3
import s3fs # for reading from S3FileSystem
import json # for working with JSON files 
import mini_utils
from datetime import datetime

import matplotlib.pyplot as plt


pd.set_option('max_colwidth', -1)<jupyter_output><empty_output><jupyter_text>Load Image CSV Info<jupyter_code>df_all = pd.read_csv('imgid_groundtruth_trainvaltest.csv')
df_all.head()
df_all.groupby(['train/val/test', 'ground_truth'])['jpg_name'].count().reset_index()
ground_truth_dict = {'0_missing' : ['only_missing'],
                     '1_present' : ['present', 'multiple_present']                 
}<jupyter_output><empty_output><jupyter_text>List for Training<jupyter_code># Combining prior "test" set with training for more data
mask_train = (df_all['train/val/test'] == 'train') | (df_all['train/val/test'] == 'validation')
df_train = df_all.loc[mask_train]
df_train_sample = mini_utils.sample_rows(df_train, 500, ground_truth_dict)
df_train_sample['train/valid'] = 'train'
df_train_sample['folder_label'].value_counts()
# Using about 15% of examples
df_val = df_all.loc[df_all['train/val/test'] == 'test']
df_val_sample = mini_utils.sample_rows(df_val, 400, ground_truth_dict)
df_val_sample['train/valid'] = 'valid'
df_val_sample['folder_label'].value_counts()
df_transfer = pd.concat([df_train_sample, df_val_sample])
print(df_transfer.shape)
df_transfer.head()
df_transfer.groupby(['train/valid', 'folder_label'])['jpg_name'].count().reset_index()
df_transfer.to_csv('missing-present-2split-img-details.csv', index = False)<jupyter_output><empty_output><jupyter_text>Specify the destination Path* jpg_name* train/val/test* folder_label<jupyter_code>df_transfer['original_image_path'] = 'mini-crops/' + df_transfer['jpg_name']
df_transfer['new_image_path'] = 'mini-crops-labeled/missing-present-2split/' + df_transfer['train/valid'] + '/' + df_transfer['folder_label'] + '/' + df_transfer['jpg_name']
show_transfer_cols = ['jpg_name', 'train/valid', 'folder_label', 'original_image_path', 'new_image_path']
df_transfer[show_transfer_cols].head()
df_transfer = df_transfer.reset_index(drop = True)
# Move files from outer folder to gsv folder
bucket_name = 'gsv-crops2'
s3 = boto3.resource('s3')

start = datetime.now()
print(start)
cannot_move_paths = []
for index, row in df_transfer.iterrows():
    if index % 500 == 0:
        print(f'Through {index} images at {datetime.now()}')
    # Copy to gsv folder
    old_name = str(row['original_image_path'])
    new_name = str(row['new_image_path'])
    copy_source = {
        'Bucket': bucket_name,
        'Key': old_name
    }
    try:
        s3.meta.client.copy(copy_source, bucket_name, new_name)
    except:
        print(f'Could not move {old_name} at {datetime.now()}')
        cannot_move_paths.append(old_name)
end = datetime.now()
print(end)
df_can_move = df_transfer.loc[~df_transfer['original_image_path'].isin(cannot_move_paths)]
df_can_move.to_csv('crossingnull-crops-successful-move.csv', index = False)
df_can_move.groupby(['train/valid', 'folder_label'])['jpg_name'].count().reset_index()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13312338963641568, 'fraction_numerical': 0.007443458345261953, 'mean_word_length': 6.058585858585858, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '637298', 'license': 'MIT', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1168, 'n_words': 242, 'path': 'mini-crops/2020-04-08-Missing-Present-2Class-Samples.ipynb', 'repo_name': 'w210-accessibility/classify-streetview', 'size': '3493'}