<commit_before>import glob
import numpy as np
import re
import sys

import pandas as pd

sys.path.append('./support_files/')
#import summarise_bins

import name_extractions


def load_individual_bin_summaries():
    return pd.read_csv('support_files/available_bins.csv', sep =',')


# def load_bin_length_summary():
#     return pd.read_csv('support_files/available_bins.csv', sep =',')


def load_one_mummer_result(filepath):
    """
    Load parsed MUMMER file (parsed into .tsv), and rename
    :param filepath:
    :return:
    """
    # trim ./mummer-results/filename.tsv to filename.tsv
    # col names: TAGS (ref)    TAGS (query)  LEN 1  LEN 2  LEN R  LEN Q
    # COV R  COV Q % IDY

    # Read in parsed MUMMER result.
    df = pd.read_csv(filepath, sep='\t')

    # check that the dataframe is not empty
    if df.shape[0] < 1:
        # return empty dataframe
        return df

    assert df.shape[0] > 0, "dataframe for {} has 0 rows".format(filepath)

    # The 2 argument says split at most 2 times.
    df['mummer file'] = filepath.split("/", 2)[2]

    df['query name'] = \
        df['mummer file'].apply(
            lambda x:
            name_extractions.query_and_ref_names_from_path(x)['query'])
    df['ref name'] = \
        df['mummer file'].apply(
            lambda x:
            name_extractions.query_and_ref_names_from_path(x)['ref'])
    # Split 'TAGS (ref)' = Ga0081624_1057 into a column called 'ref bin'
    # (= 'Ga0081624') and 'ref contig' = 1057
    # Same for 'TAGS (query)
    df['ref id'] = df['TAGS (ref)'].apply(
        lambda x: name_extractions.extract_bin_number(x))
    df['query id'] = df['TAGS (query)'].apply(
        lambda x: name_extractions.extract_bin_number(x))
    df['ref contig'] = df['TAGS (ref)'].apply(
        lambda x: name_extractions.extract_contig_number(x))
    df['query contig'] = df['TAGS (query)'].apply(
        lambda x: name_extractions.extract_contig_number(x))
    return df


def prep_summary_for_merge(df, prepend_string):
    """
    Prepend the specified string onto every column name of the dataframe.

    :param df: dataframe to alter the column names of.
    :param prepend_string: string to prpend to each column name.
    :return: dataframe with modified column names
    """
    df2 = df.copy()
    # remove the category column, which will cause problems.
    if 'category' in df2.columns:
        df2 = df2.drop('category', axis=1)
    if 'bin path' in df2.columns:
        df2 = df2.drop('bin path', axis=1)
    # prepend the string onto each column name
    df2.columns = map(lambda x: prepend_string + x, df2.columns)
    return df2


def prepare_result(filepath):
    result = load_one_mummer_result(filepath)
    if result.shape[0] == 0:
        return None

    # prepare meta-info for the query bin
    # Customize the generalized metainfo for the query, then the ref.
    query_metainfo = prep_summary_for_merge(
        df=load_individual_bin_summaries(),
        prepend_string='query ')

    # prepare meta-info for the ref bin
    ref_metainfo = prep_summary_for_merge(
        df=load_individual_bin_summaries(),
        prepend_string='ref ')

    def check_merge_failure():
        assert result.shape[0] > 0, \
            'after merging on query metainfo, you only have {} rows ' \
            'remaining. Did inner merge fail?'.format(result.shape[0])

    def check_merge_name_success():
        unique_query_bin_names = result['query name'].unique()
        unique_ref_bin_names = result['ref name'].unique()
        assert len(unique_query_bin_names) == 1, \
            'should only have one query name, but have {} for {}'.format(
                len(unique_query_bin_names), result['mummer file'][0]
            )
        assert len(unique_ref_bin_names) == 1, \
            'should only have one query name, but have {} for {}'.format(
                len(unique_ref_bin_names), result['mummer file'][0]
            )

    result = pd.merge(result, query_metainfo, how='inner', on='query name')
    check_merge_failure()

    result = pd.merge(result, ref_metainfo, how='inner', on='ref name')
    check_merge_failure()

    check_merge_name_success()

    return result


def dataframe_is_one_query_target_pair(dataframe):
    """
    make sure there is only one query sequence and reference sequence in the
    given dataframe.  Used to check that we aren't aggregating % identity
    numbers across bin alignment pairs.

    :param dataframe:
    :return:
    """
    num_query_bins = len(dataframe['query name'].unique())
    num_ref_bins = len(dataframe['ref name'].unique())<commit_msg>drop bin name, which broke for Methylobacte-98r<commit_after>import glob
import numpy as np
import re
import sys

import pandas as pd

sys.path.append('./support_files/')
#import summarise_bins

import name_extractions


def load_individual_bin_summaries():
    return pd.read_csv('support_files/available_bins.csv', sep =',')


# def load_bin_length_summary():
#     return pd.read_csv('support_files/available_bins.csv', sep =',')


def load_one_mummer_result(filepath):
    """
    Load parsed MUMMER file (parsed into .tsv), and rename
    :param filepath:
    :return:
    """
    # trim ./mummer-results/filename.tsv to filename.tsv
    # col names: TAGS (ref)    TAGS (query)  LEN 1  LEN 2  LEN R  LEN Q
    # COV R  COV Q % IDY

    # Read in parsed MUMMER result.
    df = pd.read_csv(filepath, sep='\t')

    # check that the dataframe is not empty
    if df.shape[0] < 1:
        # return empty dataframe
        return df

    assert df.shape[0] > 0, "dataframe for {} has 0 rows".format(filepath)

    # The 2 argument says split at most 2 times.
    df['mummer file'] = filepath.split("/", 2)[2]

    df['query name'] = \
        df['mummer file'].apply(
            lambda x:
            name_extractions.query_and_ref_names_from_path(x)['query'])
    df['ref name'] = \
        df['mummer file'].apply(
            lambda x:
            name_extractions.query_and_ref_names_from_path(x)['ref'])
    # Split 'TAGS (ref)' = Ga0081624_1057 into a column called 'ref bin'
    # (= 'Ga0081624') and 'ref contig' = 1057
    # Same for 'TAGS (query)
    df['ref id'] = df['TAGS (ref)'].apply(
        lambda x: name_extractions.extract_bin_number(x))
    df['query id'] = df['TAGS (query)'].apply(
        lambda x: name_extractions.extract_bin_number(x))
    df['ref contig'] = df['TAGS (ref)'].apply(
        lambda x: name_extractions.extract_contig_number(x))
    df['query contig'] = df['TAGS (query)'].apply(
        lambda x: name_extractions.extract_contig_number(x))
    return df


def prep_summary_for_merge(df, prepend_string):
    """
    Prepend the specified string onto every column name of the dataframe.

    :param df: dataframe to alter the column names of.
    :param prepend_string: string to prpend to each column name.
    :return: dataframe with modified column names
    """
    df2 = df.copy()
    # remove the bin name, which is inconsistent for Methylobact-98r
    # remove the category column, which will cause problems.
    df2 = df2.drop('name', axis = 1)
    if 'category' in df2.columns:
        df2 = df2.drop('category', axis=1)
    if 'bin path' in df2.columns:
        df2 = df2.drop('bin path', axis=1)
    # prepend the string onto each column name
    df2.columns = map(lambda x: prepend_string + x, df2.columns)
    return df2


def prepare_result(filepath):
    result = load_one_mummer_result(filepath)
    if result.shape[0] == 0:
        return None

    # prepare meta-info for the query bin
    # Customize the generalized metainfo for the query, then the ref.
    query_metainfo = prep_summary_for_merge(
        df=load_individual_bin_summaries(),
        prepend_string='query ')

    # prepare meta-info for the ref bin
    ref_metainfo = prep_summary_for_merge(
        df=load_individual_bin_summaries(),
        prepend_string='ref ')

    def check_merge_failure(merge_name):
        assert result.shape[0] > 0, \
            'after merging on {} metainfo, you only have {} rows ' \
            'remaining. Did inner merge fail?'.format(merge_name,
                                                      result.shape[0])

    def check_merge_name_success():
        unique_query_bin_names = result['query name'].unique()
        unique_ref_bin_names = result['ref name'].unique()
        assert len(unique_query_bin_names) == 1, \
            'should only have one query name, but have {} for {}'.format(
                len(unique_query_bin_names), result['mummer file'][0]
            )
        assert len(unique_ref_bin_names) == 1, \
            'should only have one query name, but have {} for {}'.format(
                len(unique_ref_bin_names), result['mummer file'][0]
            )

    result = pd.merge(result, query_metainfo, how='inner', on='query id')
    check_merge_failure('result')

    result = pd.merge(result, ref_metainfo, how='inner', on='ref id')
    check_merge_failure('query')

    check_merge_name_success()

    return result


def dataframe_is_one_query_target_pair(dataframe):
    """
    make sure there is only one query sequence and reference sequence in the
    given dataframe.  Used to check that we aren't aggregating % identity
    numbers across bin alignment pairs.

    :param dataframe:
    :return:
    """
    num_query_bins = len(dataframe['query name'].unique())
    num_ref_bins = len(dataframe['ref name'].unique())
{'fraction_non_alphanumeric': 0.11210002119093028, 'fraction_numerical': 0.01218478491205764, 'mean_word_length': 2.9214790195263816, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '9603', 'n_tokens_mistral': 3115, 'n_tokens_neox': 2964, 'n_words': 830}