<commit_before>from glider_utils.yo import create_profile_entry


def filter_profiles(profiles, dataset, conditional):
    """Filters out profiles that do not meet some criteria

    Returns the filtered set of profiles
    """

    filtered_profiles = []
    start_index = 0
    for profile in profiles:
        end_index = profile['index_bounds'][1]
        if conditional(profile):
            filtered_profiles.append(
                create_profile_entry(
                    dataset,
                    start_index,
                    end_index
                )
            )
            start_index = end_index
        elif len(dataset) == end_index:
            filtered_profiles.append(
                create_profile_entry(
                    dataset,
                    start_index,
                    end_index
                )
            )

    return filtered_profiles

# Convenience methods follow


def filter_profile_depth(profiles, dataset, below=1):
    """Filters out profiles that are not below a certain depth (Default: 1m)

    Returns the filtered set of profiles<commit_msg>Fixed issue with filters allowing last points during surfacing to be own profile despite profile length filter<commit_after>from glider_utils.yo import create_profile_entry


def filter_profiles(profiles, dataset, conditional):
    """Filters out profiles that do not meet some criteria

    Returns the filtered set of profiles
    """

    filtered_profiles = []
    start_index = 0
    for profile in profiles:
        end_index = profile['index_bounds'][1]
        if conditional(profile):
            filtered_profiles.append(
                create_profile_entry(
                    dataset,
                    start_index,
                    end_index
                )
            )
            start_index = end_index
        elif len(dataset) == end_index:
            # Should this one be merged with the last
            # or made into its own?
            if conditional(profile):
                filtered_profiles.append(
                    create_profile_entry(
                        dataset,
                        start_index,
                        end_index
                    )
                )
            elif len(filtered_profiles) > 0:
                last_bounds = filtered_profiles[-1]['index_bounds']
                filtered_profiles[-1] = (
                    create_profile_entry(
                        dataset,
                        last_bounds[0],
                        end_index
                    )
                )
            else:
                filtered_profiles.append(
                    create_profile_entry(
                        dataset,
                        start_index,
                        end_index
                    )
                )

    return filtered_profiles

# Convenience methods follow


def filter_profile_depth(profiles, dataset, below=1):
    """Filters out profiles that are not below a certain depth (Default: 1m)

    Returns the filtered set of profiles
{'fraction_non_alphanumeric': 0.05366492146596859, 'fraction_numerical': 0.003926701570680628, 'mean_word_length': 1.4281175536139794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4302020', 'n_tokens_mistral': 737, 'n_tokens_neox': 682, 'n_words': 198}