<commit_before>def pull_S3_sublist(yaml_outpath, img_type, bucket_name, bucket_prefix, \
                        creds_path, include_site=False, series_list=None):

    import os
    import yaml
    from indi_aws import fetch_creds

    s3_list = []
    s3_dict = {}

    bucket = fetch_creds.return_bucket(creds_path, bucket_name)

    # Filter for anat/rest
    if img_type == 'anat':
        subkey_type = 'anatomical_scan'
    elif img_type == 'func':
        subkey_type = 'functional_scan'


    # Build S3-subjects to download
    for bk in bucket.objects.filter(Prefix=bucket_prefix):
        s3_list.append(str(bk.key))


    # Read in series_list, if it is provided
    if series_list:
        try:
            series_list = os.path.abspath(series_list)
            with open(series_list,"r") as f:
                series = f.readlines()
            series = [i.rstrip("\n") for i in series]
        except:
            err = "\n\nCould not successfully read the series list.\n%s" \
                  % series_list
            raise Exception(err)


    # Build dictionary of filepaths
    for sfile in s3_list:

        ssplit = sfile.split('/')

        sub_id = ssplit[-4]

        session_id = ssplit[-3]

        scan_type = ssplit[-2]

        filename = ssplit[-1]

        if include_site:
            site_id = ssplit[-5]

        if (img_type == scan_type) and ("nii" in filename):
        
            resource_dict = {}
            resource_dict[subkey_type] = sfile

            if incude_site:
                resource_dict["site_name"] = site_id

            if series_list:
                selected = 0
                for series_name in series:
                    if series_name in filename:
                        selected = 1
                if selected == 0:
                    continue

            # this ONLY handles raw data inputs, not CPAC-generated outputs!
            if not s3_dict.has_key((sub_id, session_id, filename)):

                s3_dict[(sub_id, session_id, filename)] = {}
                s3_dict[(sub_id, session_id, filename)].update(resource_dict)

            else:

                s3_dict[(sub_id, session_id, filename)].update(resource_dict)    <commit_msg>Made the AWS s3 dict generator more robust to varying series and filename names.<commit_after>def pull_S3_sublist(yaml_outpath, img_type, bucket_name, bucket_prefix, \
                        creds_path, include_site=False, series_list=None):

    import os
    import yaml
    from indi_aws import fetch_creds

    s3_list = []
    s3_dict = {}

    bucket = fetch_creds.return_bucket(creds_path, bucket_name)

    # Filter for anat/rest
    if img_type == 'anat':
        subkey_type = 'anatomical_scan'
    elif img_type == 'func':
        subkey_type = 'functional_scan'


    # Build S3-subjects to download
    for bk in bucket.objects.filter(Prefix=bucket_prefix):
        s3_list.append(str(bk.key))


    # Read in series_list, if it is provided
    if series_list:
        try:
            series_list = os.path.abspath(series_list)
            with open(series_list,"r") as f:
                series = f.readlines()
            series = [i.rstrip("\n") for i in series]
        except:
            err = "\n\nCould not successfully read the series list.\n%s" \
                  % series_list
            raise Exception(err)


    # Build dictionary of filepaths
    for sfile in s3_list:

        ssplit = sfile.split('/')

        sub_id = ssplit[-4]

        session_id = ssplit[-3]

        scan_type = ssplit[-2]

        filename = ssplit[-1]

        if include_site:
            site_id = ssplit[-5]

        include = False
        if img_type == "anat":
            if ("anat" in scan_type) or ("anat" in filename) or \
                ("mprage" in filename):
                include = True
        if img_type == "func":
            if ("func" in scan_type) or ("rest" in scan_type) or \
                ("func" in filename) or ("rest" in filename):
                include = True

        if (include == True) and ("nii" in filename):
        
            resource_dict = {}
            resource_dict[subkey_type] = sfile

            if incude_site:
                resource_dict["site_name"] = site_id

            if series_list:
                selected = 0
                for series_name in series:
                    if series_name in filename:
                        selected = 1
                if selected == 0:
                    continue

            # this ONLY handles raw data inputs, not CPAC-generated outputs!
            if not s3_dict.has_key((sub_id, session_id, filename)):

                s3_dict[(sub_id, session_id, filename)] = {}
                s3_dict[(sub_id, session_id, filename)].update(resource_dict)

            else:

                s3_dict[(sub_id, session_id, filename)].update(resource_dict)    
{'fraction_non_alphanumeric': 0.09051812410403441, 'fraction_numerical': 0.007577309031333196, 'mean_word_length': 1.7361344537815127, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '393117', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1401, 'n_words': 367}