<jupyter_start><jupyter_text>Read in train data<jupyter_code>bucket = 'assistive-vision'
subfolder = 's3://assistive-vision/TextCaps/train/'
from sagemaker import get_execution_role
role = get_execution_role()
role
conn = boto3.client('s3')
contents = conn.list_objects(Bucket=bucket, Prefix=subfolder)['Contents']
for f in contents:
    print(f['Key'])
role = get_execution_role()
bucket='assistive-vision'
data_key = 'train.csv'
data_location = 's3://assistive-vision/TextCaps/train/'.format(bucket, data_key)

pd.read_csv(data_location)
import s3fs
fs = s3fs.S3FileSystem()

# To List 5 files in your accessible bucket
fs.ls('s3://assistive-vision/TextCaps/train/')[:5]

# open it directly
with fs.open(f's3://assistive-vision/TextCaps/train/0000e8b36676338b.jpg') as f:
    display(Image.open(f))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14337349397590363, 'fraction_numerical': 0.02891566265060241, 'mean_word_length': 6.419642857142857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '143409', 'license': 'MIT', 'n_tokens_mistral': 326, 'n_tokens_neox': 311, 'n_words': 54, 'path': 'TextCapsEDA.ipynb', 'repo_name': 'sony-w/AssistiveVision', 'size': '830'}