<commit_before>import pandas as pd
import load_data

from sklearn.feature_extraction import DictVectorizer

FEATURES_TO_EXTRACT = ['kingdom', 'phylum', 'class', 'order',
                       'family', 'genus', 'length', 'oxygen',
                       'replicate', 'week', 'abundance']


def phylo_levels_below(phylo_level):
    """
    E.g. 'order' --> ['family', 'genus']
    """
    p_levels = ['kingdom', 'phylum', 'class', 'order', 'family', 'genus']
    position_of_phylo_level = p_levels.index(phylo_level)
    return p_levels[position_of_phylo_level + 1:]


def phylo_levels_above(phylo_level):
    """
    E.g. 'order' --> ['kingdom', 'phylum', 'class']
    """
    p_levels = ['kingdom', 'phylum', 'class', 'order', 'family', 'genus']
    position_of_phylo_level = p_levels.index(phylo_level)
    return p_levels[0:position_of_phylo_level]


def aggregate_on_phylo_level(dataframe, phylo_level):
    # If you pass ['kingdom', 'phylum', 'class'], it will return a
    # dataframe with only those phylo levels specified and all lower levels
    # All abundances for that set of phylo levels is aggregated and the
    # abundances for each condtion are summed.
    # dropp all phylo_levels below the specified one.

    # Groupby the project (includes week, O2)
    if 'length' in dataframe.columns:
        dataframe.drop('length', axis=1, inplace=True)
    groupby_levels = ['oxygen', 'replicate', 'week'] + \
                     phylo_levels_above(phylo_level) + [phylo_level]
    return dataframe.groupby(groupby_levels).sum()


def check_abundances_sums(dataframe):
    # todo: make a function that checks that each sample's abundances sum to
#  1.
    pass


def filter_by_abundance(dataframe, low, high=1, abundance_column='abundance',
                        phylo_column='genus'):
    """
    Return only rows where the specified phylo_colum's set of rows have at
    least one value of abundance_column in range(low, high)

    E.g. only return the genus Methylobacter if at least one sample in the
    data has abundance over the specified `low` input.

    :param data: dataframe to reduce
    :param abundance_column: column to filter by.  Defalts to 'abundance'
    :param phylo_column: column to grab unique values from.  Defaults to
    'genus'.
    :param low: lowest abundance to look for
    :param high: highest abundance to look for (default = 1)
    :return: dataframe
    """
    # todo: assert that the desired abundance and phylo columns exist.

    # get a list of the phylo_column names that meet our criteria.
    phylo_column_values_to_keep = \
        dataframe[(dataframe[abundance_column] <= high) &
                  (dataframe[abundance_column] >= low)][phylo_column].unique()
    print("first (up to) 5 phylo columns to "
          "keep: {}".format(phylo_column_values_to_keep[0:5]))
    # Return ALL rows for a phylo_column label if ANY of the rows had an
    # abundance value in the desired range.
    return dataframe[dataframe[phylo_column].isin(phylo_column_values_to_keep)]


def reduce_data(dataframe, min_abundance, phylo_column='genus', oxygen="all"):
    # note: we are aggregating on phylo level *before* filtering by
    # abundance.  This choice impacts the output!
    # todo: fill NA values as "other"?
    # Consider only the desired oxygen condition(s)
    if (oxygen == "Low") or (oxygen == 'low'):
        dataframe = dataframe[dataframe['oxygen'] == 'Low']
    if (oxygen == "High") or (oxygen == "high"):
        dataframe = dataframe[dataframe['oxygen'] == 'High']
    # todo: make sure only the right oxygen condition was selected.

    # aggregate on the desired phylogenetic level.<commit_msg>PEP8 changes<commit_after>import pandas as pd
import load_data

from sklearn.feature_extraction import DictVectorizer

FEATURES_TO_EXTRACT = ['kingdom', 'phylum', 'class', 'order',
                       'family', 'genus', 'length', 'oxygen',
                       'replicate', 'week', 'abundance']


def phylo_levels_below(phylo_level):
    """
    E.g. 'order' --> ['family', 'genus']
    """
    p_levels = ['kingdom', 'phylum', 'class', 'order', 'family', 'genus']
    position_of_phylo_level = p_levels.index(phylo_level)
    return p_levels[position_of_phylo_level + 1:]


def phylo_levels_above(phylo_level):
    """
    E.g. 'order' --> ['kingdom', 'phylum', 'class']
    """
    p_levels = ['kingdom', 'phylum', 'class', 'order', 'family', 'genus']
    position_of_phylo_level = p_levels.index(phylo_level)
    return p_levels[0:position_of_phylo_level]


def aggregate_on_phylo_level(dataframe, phylo_level):
    # If you pass ['kingdom', 'phylum', 'class'], it will return a
    # dataframe with only those phylo levels specified and all lower levels
    # All abundances for that set of phylo levels is aggregated and the
    # abundances for each condtion are summed.
    # dropp all phylo_levels below the specified one.

    # Groupby the project (includes week, O2)
    if 'length' in dataframe.columns:
        dataframe.drop('length', axis=1, inplace=True)
    groupby_levels = ['oxygen', 'replicate', 'week'] + \
                     phylo_levels_above(phylo_level) + [phylo_level]
    return dataframe.groupby(groupby_levels).sum()


def check_abundances_sums(dataframe):
    # todo: make a function that checks that each sample's abundances sum to
#  1.
    pass


def filter_by_abundance(dataframe, low, high=1, abundance_column='abundance',
                        phylo_column='genus'):
    """
    Return only rows where the specified phylo_colum's set of rows have at
    least one value of abundance_column in range(low, high)

    E.g. only return the genus Methylobacter if at least one sample in the
    data has abundance over the specified `low` input.

    :param data: dataframe to reduce
    :param abundance_column: column to filter by.  Defalts to 'abundance'
    :param phylo_column: column to grab unique values from.  Defaults to
    'genus'.
    :param low: lowest abundance to look for
    :param high: highest abundance to look for (default = 1)
    :return: dataframe
    """
    # todo: assert that the desired abundance and phylo columns exist.

    # get a list of the phylo_column names that meet our criteria.
    phylo_column_values_to_keep = \
        dataframe[(dataframe[abundance_column] <= high) &
                  (dataframe[abundance_column] >= low)][phylo_column].unique()
    print "first (up to) 5 phylo columns to "
    print "keep: {}".format(phylo_column_values_to_keep[0:5])
    # Return ALL rows for a phylo_column label if ANY of the rows had an
    # abundance value in the desired range.
    return dataframe[dataframe[phylo_column].isin(phylo_column_values_to_keep)]


def reduce_data(dataframe, min_abundance, phylo_column='genus', oxygen="all"):
    # note: we are aggregating on phylo level *before* filtering by
    # abundance.  This choice impacts the output!
    # todo: fill NA values as "other"?
    # Consider only the desired oxygen condition(s)
    if (oxygen == "Low") or (oxygen == 'low'):
        dataframe = dataframe[dataframe['oxygen'] == 'Low']
    if (oxygen == "High") or (oxygen == "high"):
        dataframe = dataframe[dataframe['oxygen'] == 'High']
    # todo: make sure only the right oxygen condition was selected.

    # aggregate on the desired phylogenetic level.
{'fraction_non_alphanumeric': 0.10859481582537517, 'fraction_numerical': 0.00286493860845839, 'mean_word_length': 3.4701219512195123, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3700690', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2209, 'n_words': 718}