<commit_before>import pandas as pd


# # XXX PAML no longer needs a format different than `Homo_sapiens`
def paml_org_formatter(organisms):
    org_list = []
    for organism in organisms:
        genus, sep, species = organism.partition('_')
        org = ''.join([genus[0], sep, species[0:28]])
        org_list.append(org)
    return org_list

# ***********************************************PRE BLAST ANALYSIS TOOLS********************************************* #
# ***********************************************PRE BLAST ANALYSIS TOOLS********************************************* #


def my_gene_info(acc_path, blast_query='Homo_sapiens'):
    import mygene

    # Initialize variables and import my-gene search command
    urls = []
    df = pd.read_csv(str(acc_path), dtype=str)
    df_gene = df.set_index('Gene')
    blast_query_list = df_gene[blast_query].tolist()
    mg = mygene.MyGeneInfo()

    # Create a my-gene query handle to get the data
    human = list(x.upper() for x in blast_query_list)
    mygene_query = mg.querymany(human, scopes='refseq',
                                fields='symbol,name,entrezgene,summary',
                                species='human', returnall=True, as_dataframe=True,
                                size=1, verbose=True)
    # TODO-ROB:  Logging here
    # Turn my-gene queries into a data frame and then reset the index
    mygene_query['out'].reset_index(level=0, inplace=True)
    mg_df = pd.DataFrame(mygene_query['out'])
    mg_df.drop(mg_df.columns[[1, 2, 6]], axis=1, inplace=True)
    # Rename the columns
    mg_df.rename(columns={'entrezgene': 'Entrez ID', 'summary':
                         'Gene Summary', 'query': 'RefSeqRNA Accession', 'name': 'Gene Name'},
                 inplace=True)

    # Create NCBI links using a for loop and the Entrez IDs
    urls = [('<a href="{0}">{0}</a>'.format('https://www.ncbi.nlm.nih.gov/gene/' + str(entrez_id)))
            for entrez_id in mg_df['Entrez ID']]

    ncbi = pd.DataFrame(urls, columns=['NCBI Link'], dtype=str)
    # Merge, sort, and return the my-gene data frame

    hot_data = pd.concat([pd.Series(df.Tier, dtype=str), df.Gene, mg_df, ncbi], axis=1)
    hot_data.rename(columns={'Gene': 'Gene Symbol'}, inplace=True)
    hot_data = hot_data.sort_values(['Tier'], ascending=True)

    return hot_data
# **********************************************POST BLAST ANALYSIS TOOLS******************************************** #
# **********************************************POST BLAST ANALYSIS TOOLS******************************************** #


def get_dup_acc(acc_dict, gene_list, org_list):<commit_msg>Added and changed gene_list_config() to the utils file.<commit_after>import pandas as pd
import os
import csv
from pathlib import Path
import time


def map_func(hit):
    """Use the map function for formatting hit id's.
    This will be used later in the script.
    """
    hit.id1 = hit.id.split('|')[3]  # accession number
    hit.id2 = hit.id.split('|')[1]  # gi number
    hit.id = hit.id[:-2]
    return hit


# # XXX PAML no longer needs a format different than `Homo_sapiens`
def paml_org_formatter(organisms):
    org_list = []
    for organism in organisms:
        genus, sep, species = organism.partition('_')
        org = ''.join([genus[0], sep, species[0:28]])
        org_list.append(org)
    return org_list


def gene_list_config(file, data_path, gene_list, taxon_dict, logger):
    """Create or use a blast configuration file.
    This function configures different files for new BLASTS.
    It also helps recognize whether or not a BLAST was terminated
    in the middle of the dataset.  This removes the last line of
    the accession file if it is incomplete.
    """

    ending = gene = org = taxid = None
    output_dir_list = os.listdir(data_path)  # Make a list of files
    # If the file exists then make a gene list that picks up from the last BLAST
    if file in output_dir_list:
        header = pd.read_csv(str(Path(data_path) / Path(file)), dtype=str)
        header = header.axes[1].tolist()
        file = Path(data_path) / Path(file)
        with open(str(file), 'r') as open_file:
            csv_file = csv.reader(open_file)
            count = 0
            # Iterate the csv file and determine the last gene to be blasted
            for row in csv_file:
                if count == 0:
                    header = row
                count += 1
                ending = row  # The last row
                gene = ending[1]  # The last row's gene
                org = header[len(row) - 1]  # The last column(organism) accessed in the last row
                taxid = taxon_dict[org]  # The taxon id of the organism

            # ######### Start logging ######### #
            ncbi = str("""result_handle1 = NcbiblastnCommandline(query="temp.fasta", db="refseq_rna", strand="plus",
                evalue=0.001, out="%s_%s.xml", outfmt=5, gilist=%s + "gi", max_target_seqs=10, task="blastn")"""
                       % (gene, org, taxid))
            logger.warning("An incomplete accession file was produced from the previous BLAST, which was terminated "
                           "midway through the procedure.")
            logger.info("The last row looks like: \n\t%s\n\t%s\n" % (header, ending))
            logger.info("The BLAST ended on the following query: \n%s" % ncbi)
            if len(ending) < len(header):
                logger.info("Restarting the BLAST for the previous gene...")
                count = count - 2
            # ######## End Logging ######## #
            # The continued gene list starts with the previous gene.
            continued_gene_list = list(x for i, x in enumerate(gene_list, 1) if i > count)
        return continued_gene_list
    # If the file doesn't exist return nothing
    else:
        logger.info("A new BLAST started at %s" % time.time())
        return None
# ***********************************************PRE BLAST ANALYSIS TOOLS********************************************* #
# ***********************************************PRE BLAST ANALYSIS TOOLS********************************************* #


def my_gene_info(acc_path, blast_query='Homo_sapiens'):
    import mygene

    # Initialize variables and import my-gene search command
    urls = []
    df = pd.read_csv(str(acc_path), dtype=str)
    df_gene = df.set_index('Gene')
    blast_query_list = df_gene[blast_query].tolist()
    mg = mygene.MyGeneInfo()

    # Create a my-gene query handle to get the data
    human = list(x.upper() for x in blast_query_list)
    mygene_query = mg.querymany(human, scopes='refseq',
                                fields='symbol,name,entrezgene,summary',
                                species='human', returnall=True, as_dataframe=True,
                                size=1, verbose=True)
    # TODO-ROB:  Logging here
    # Turn my-gene queries into a data frame and then reset the index
    mygene_query['out'].reset_index(level=0, inplace=True)
    mg_df = pd.DataFrame(mygene_query['out'])
    mg_df.drop(mg_df.columns[[1, 2, 6]], axis=1, inplace=True)
    # Rename the columns
    mg_df.rename(columns={'entrezgene': 'Entrez ID', 'summary': 'Gene Summary', 'query': 'RefSeqRNA Accession',
                          'name': 'Gene Name'},
                 inplace=True)

    # Create NCBI links using a for loop and the Entrez IDs
    urls = [('<a href="{0}">{0}</a>'.format('https://www.ncbi.nlm.nih.gov/gene/' + str(entrez_id)))
            for entrez_id in mg_df['Entrez ID']]

    ncbi = pd.DataFrame(urls, columns=['NCBI Link'], dtype=str)
    # Merge, sort, and return the my-gene data frame

    hot_data = pd.concat([pd.Series(df.Tier, dtype=str), df.Gene, mg_df, ncbi], axis=1)
    hot_data.rename(columns={'Gene': 'Gene Symbol'}, inplace=True)
    hot_data = hot_data.sort_values(['Tier'], ascending=True)

    return hot_data
# **********************************************POST BLAST ANALYSIS TOOLS******************************************** #
# **********************************************POST BLAST ANALYSIS TOOLS******************************************** #


def get_dup_acc(acc_dict, gene_list, org_list):
{'fraction_non_alphanumeric': 0.19796643390910204, 'fraction_numerical': 0.005757687124831557, 'mean_word_length': 3.003923491907798, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2996480', 'n_tokens_mistral': 2571, 'n_tokens_neox': 2390, 'n_words': 679}