<commit_before>#! /usr/bin/env python2

from sys import exit
import pickle
from difflib import SequenceMatcher
from itertools import islice
from RecBlastUtils import *


all_genes_dict = {}


# find similarity between sequences
def similar(a, b):
    """
    Evaluates similarity ratio between a and b.
    :param a: string
    :param b: string
    :return: similarity ratio
    """
    return SequenceMatcher(None, a, b).ratio()


def format_description(description, regex, species):
    """
    Reformat the description for consistency.
    """
    species = lower(str(species))
    description = replace(lower(str(description)), "predicted: ", "")
    description = replace(description, "[{}]".format(species), "")
    description = replace(description, "{}".format(species), "")
    description = replace(description, "unnamed protein product", "")
    description = replace(description, "isoform", "")
    description = strip(re_sub(regex, '', description))
    return description


# structure of update_match_results:
# {gene_name: { animal1: [ [strict, not strict] ],  animal2: [],  animal3 = [], ...}}
def update_match_results(enumerator, animal_org, target_name1, this_gene_dic, is_rbh, DEBUG, debug):
    """
    Prints final output for each gene. Saves the output to this_gene_dic, but also returns a status which will be used
    later to determine if the match is RBH/strict/non-strict.
    :param is_rbh: True if the value is RBH, False if not.
    :param this_gene_dic:
    :param enumerator: result number for gene_id
    :param animal_org:  Name of the organism
    :param target_name1:  Target gene name
    :param DEBUG:  The debug flag (True/False)
    :param debug:  The debug function
    :return:
    """

    status = "non-strict"  # default value
    try:  # update current animal
        this_gene_dic[animal_org][1] += 1  # non strict increments anyway
        if enumerator == 1:  # strict increment
            this_gene_dic[animal_org][0] += 1
            status = "strict"

    except KeyError:  # if we don't have animal_org in our db
        if enumerator == 1:  # if it is the first match
            if is_rbh:
                this_gene_dic[animal_org] = [1, 1, 1]
                status = "RBH"
            else:
                this_gene_dic[animal_org] = [1, 1, 0]
                status = "strict"
        elif enumerator > 1:  # found match, not first
            this_gene_dic[animal_org] = [0, 1, 0]

    debug("current status for gene %s, animal %s: %s" % (target_name1, animal_org, this_gene_dic[animal_org]))
    return status


def write_all_output_csv(out_dict, org_list, csv_rbh_output_filename, csv_strict_output_filename,
                         csv_ns_output_filename, DEBUG, debug, good_tax_list):
    """
    Writes formatted final output to CSV.
    :param out_dict: a dictionary of dictionaries
    :param org_list: a list of organisms
    :param csv_rbh_output_filename: the output file path of the RBH csv
    :param csv_strict_output_filename: the output file path of the strict csv
    :param csv_ns_output_filename: the output file path of the non-strict csv
    :param good_tax_list: a list of the currently good taxa (for validation)
    :param DEBUG: the DEBUG flag (True for debug mode)
    :param debug: the debug function
    :return:
    """

    # get all organisms
    all_org_list = sorted(list(set(org_list + good_tax_list)))
    # open 3 files:
    header_line = "{}\n".format(",".join(["gene_name"] + all_org_list))
    with open(csv_rbh_output_filename, 'w') as csv_rbh_file:
        csv_rbh_file.write(header_line)  # write header
        with open(csv_strict_output_filename, 'w') as csv_strict_file:
            csv_strict_file.write(header_line)  # write header
            with open(csv_ns_output_filename, 'w') as csv_nonstrict_file:
                csv_nonstrict_file.write(header_line)  # write header

                for gene_name in out_dict:  # write every line
                    assert type(gene_name) == str, "Gene name is not a string!"
                    # initialize output line: each line starts with the gene name
                    out_line_rbh = [gene_name]
                    out_line_strict = [gene_name]
                    out_line_non_strict = [gene_name]
                    for org in all_org_list:
                        try:  # add results to all 3 output csv files
                            debug(out_dict[gene_name][org])
                            # out_line.append(out_dict[gene_name][org])
                            results = out_dict[gene_name][org]  # list
                            out_line_rbh.append(str(results[2]))
                            out_line_strict.append(str(results[0]))
                            out_line_non_strict.append(str(results[1]))
                        # if the value doesn't exist for the animal - leave it empty
                        except KeyError:
                            # out_line.append("")
                            out_line_rbh.append("0")
                            out_line_strict.append("0")
                            out_line_non_strict.append("0")

                    # writing to each line:
                    csv_rbh_file.write("{}\n".format(",".join(out_line_rbh)))
                    csv_strict_file.write("{}\n".format(",".join(out_line_strict)))
                    csv_nonstrict_file.write("{}\n".format(",".join(out_line_non_strict)))
                    debug("Printed gene_name {0} to output files.".format(gene_name))

    return True


def get_definition(accession_list, DEBUG, debug, attempt_no=0):
    """
    Receives a list of gis and returns a list of Entrez records.
    :param accession_list: a list of accession numbers (or GIs)
    :param attempt_no: How many times did we try already? (default: 0)
    :param DEBUG: the DEBUG flag
    :param debug: the debug function
    :return:
    """
    try:  # fetching from entrez
        handle = Entrez.efetch(db="nucleotide", id=accession_list, retmode="xml")
        records = Entrez.read(handle)
        debug("Successfully connected to Entrez.")
        # get the definition and sequence of each gene
        results = [(res["GBSeq_definition"], res["GBSeq_sequence"]) for res in records]
        return results

    except Exception, e:
        debug("Error: {}\nError connecting to server, trying again...".format(e))

        # sleeping in case it's a temporary database problem
        sleep_period = 180
        print "restarting attempt in {} seconds...".format(sleep_period)
        sleep(sleep_period)

        # counting the number of attempts to connect.
        attempt_no += 1
        if attempt_no >= 10:  # If too many:
            print "Tried connecting to Entrez DB more than 10 times. Check your connection or try again later."
            exit(1)
<commit_msg>updated nucleotide database to protein<commit_after>#! /usr/bin/env python2

from sys import exit
import pickle
from difflib import SequenceMatcher
from itertools import islice
from RecBlastUtils import *


all_genes_dict = {}


# find similarity between sequences
def similar(a, b):
    """
    Evaluates similarity ratio between a and b.
    :param a: string
    :param b: string
    :return: similarity ratio
    """
    return SequenceMatcher(None, a, b).ratio()


def format_description(description, regex, species):
    """
    Reformat the description for consistency.
    """
    species = lower(str(species))
    description = replace(lower(str(description)), "predicted: ", "")
    description = replace(description, "[{}]".format(species), "")
    description = replace(description, "{}".format(species), "")
    description = replace(description, "unnamed protein product", "")
    description = replace(description, "isoform", "")
    description = strip(re_sub(regex, '', description))
    return description


# structure of update_match_results:
# {gene_name: { animal1: [ [strict, not strict] ],  animal2: [],  animal3 = [], ...}}
def update_match_results(enumerator, animal_org, target_name1, this_gene_dic, is_rbh, DEBUG, debug):
    """
    Prints final output for each gene. Saves the output to this_gene_dic, but also returns a status which will be used
    later to determine if the match is RBH/strict/non-strict.
    :param is_rbh: True if the value is RBH, False if not.
    :param this_gene_dic:
    :param enumerator: result number for gene_id
    :param animal_org:  Name of the organism
    :param target_name1:  Target gene name
    :param DEBUG:  The debug flag (True/False)
    :param debug:  The debug function
    :return:
    """

    status = "non-strict"  # default value
    try:  # update current animal
        this_gene_dic[animal_org][1] += 1  # non strict increments anyway
        if enumerator == 1:  # strict increment
            this_gene_dic[animal_org][0] += 1
            status = "strict"

    except KeyError:  # if we don't have animal_org in our db
        if enumerator == 1:  # if it is the first match
            if is_rbh:
                this_gene_dic[animal_org] = [1, 1, 1]
                status = "RBH"
            else:
                this_gene_dic[animal_org] = [1, 1, 0]
                status = "strict"
        elif enumerator > 1:  # found match, not first
            this_gene_dic[animal_org] = [0, 1, 0]

    debug("current status for gene %s, animal %s: %s" % (target_name1, animal_org, this_gene_dic[animal_org]))
    return status


def write_all_output_csv(out_dict, org_list, csv_rbh_output_filename, csv_strict_output_filename,
                         csv_ns_output_filename, DEBUG, debug, good_tax_list):
    """
    Writes formatted final output to CSV.
    :param out_dict: a dictionary of dictionaries
    :param org_list: a list of organisms
    :param csv_rbh_output_filename: the output file path of the RBH csv
    :param csv_strict_output_filename: the output file path of the strict csv
    :param csv_ns_output_filename: the output file path of the non-strict csv
    :param good_tax_list: a list of the currently good taxa (for validation)
    :param DEBUG: the DEBUG flag (True for debug mode)
    :param debug: the debug function
    :return:
    """

    # get all organisms
    all_org_list = sorted(list(set(org_list + good_tax_list)))
    # open 3 files:
    header_line = "{}\n".format(",".join(["gene_name"] + all_org_list))
    with open(csv_rbh_output_filename, 'w') as csv_rbh_file:
        csv_rbh_file.write(header_line)  # write header
        with open(csv_strict_output_filename, 'w') as csv_strict_file:
            csv_strict_file.write(header_line)  # write header
            with open(csv_ns_output_filename, 'w') as csv_nonstrict_file:
                csv_nonstrict_file.write(header_line)  # write header

                for gene_name in out_dict:  # write every line
                    assert type(gene_name) == str, "Gene name is not a string!"
                    # initialize output line: each line starts with the gene name
                    out_line_rbh = [gene_name]
                    out_line_strict = [gene_name]
                    out_line_non_strict = [gene_name]
                    for org in all_org_list:
                        try:  # add results to all 3 output csv files
                            debug(out_dict[gene_name][org])
                            # out_line.append(out_dict[gene_name][org])
                            results = out_dict[gene_name][org]  # list
                            out_line_rbh.append(str(results[2]))
                            out_line_strict.append(str(results[0]))
                            out_line_non_strict.append(str(results[1]))
                        # if the value doesn't exist for the animal - leave it empty
                        except KeyError:
                            # out_line.append("")
                            out_line_rbh.append("0")
                            out_line_strict.append("0")
                            out_line_non_strict.append("0")

                    # writing to each line:
                    csv_rbh_file.write("{}\n".format(",".join(out_line_rbh)))
                    csv_strict_file.write("{}\n".format(",".join(out_line_strict)))
                    csv_nonstrict_file.write("{}\n".format(",".join(out_line_non_strict)))
                    debug("Printed gene_name {0} to output files.".format(gene_name))

    return True


def get_definition(accession_list, DEBUG, debug, attempt_no=0):
    """
    Receives a list of gis and returns a list of Entrez records.
    :param accession_list: a list of accession numbers (or GIs)
    :param attempt_no: How many times did we try already? (default: 0)
    :param DEBUG: the DEBUG flag
    :param debug: the debug function
    :return:
    """
    try:  # fetching from entrez
        handle = Entrez.efetch(db="protein", id=accession_list, retmode="xml")
        records = Entrez.read(handle)
        debug("Successfully connected to Entrez.")
        # get the definition and sequence of each gene
        results = [(res["GBSeq_definition"], res["GBSeq_sequence"]) for res in records]
        return results

    except Exception, e:
        debug("Error: {}\nError connecting to server, trying again...".format(e))

        # sleeping in case it's a temporary database problem
        sleep_period = 180
        print "restarting attempt in {} seconds...".format(sleep_period)
        sleep(sleep_period)

        # counting the number of attempts to connect.
        attempt_no += 1
        if attempt_no >= 10:  # If too many:
            print "Tried connecting to Entrez DB more than 10 times. Check your connection or try again later."
            exit(1)

{'fraction_non_alphanumeric': 0.09438924605493863, 'fraction_numerical': 0.0062828755113968435, 'mean_word_length': 2.4060711619805923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169055', 'n_tokens_mistral': 4128, 'n_tokens_neox': 3867, 'n_words': 1188}