<commit_before>import os
import sys

import re


def roles_of_function(role):
    """
    Separate a function into a set of roles.

    In the SEED a function associated with a PEG can have more than one role. This is denoted by joining roles
    with either ' / ' (a multi-domain, multifunctional gene), ' ; ' (an ambiguous function), or ' @ '
    (a Single domain playing multiple roles). For more information see
    http://www.nmpdr.org/FIG/Html/SEED_functions.html

    This method just takes a function and returns a set of the role(s). If there is a single role, it is still a set.

    :param role: The functional role
    :type role: str
    :return: A set of the roles
    :rtype: set
    """

    # remove flanking ", comments from functions, and split multiple functions
    func = re.sub('^"|"$|\s+[#!]\s.*$', '', role)
    return set(re.split('\s*;\s+|\s+[;/@]\s+', func))


def read_features_file(features_file, verbose=False):
    """
    Read a PATRIC features file and return a set of the roles
    :param features_file: The features file to read
    :param verbose: more output
    :return: a set of the foles
    :rtype: set[str]
    """
    if not os.path.exists(features_file):
        raise IOError(f"ERROR: {features_file} does not exist")
    roles = set()
    with open(features_file, 'r') as f:
        for li in f:
            p = li.split("\t")
            if len(p) != 5:
                raise ValueError(f"{features_file} has {len(p)} columns, we were expecting 5 columns")
            for ro in roles_of_function(p[3]):
                roles.add(ro)
    return roles


def read_functional_roles(functional_roles_file, verbose=False):
    """
    Read a functional roles file and return a list of roles.
    :param functional_roles_file: the file with the roles
    :param verbose: more output
    :return:
    """
    if not os.path.exists(functional_roles_file):
        raise IOError(f"ERROR: {functional_roles_file} does not exist")
    roles = set()
    with open(functional_roles_file, 'r') as f:
        for li in f:
            for ro in roles_of_function(li):
                roles.add(ro)
    return roles


def read_downloaded_data(spreadsheet_file):
    """
    Read data downloaded from RAST as a 'spreadsheet (tab-separated text format)' and return a hash of the
    proteins and functional roles in the genome.
    :param spreadsheet_file:The file downloaded from RAST<commit_msg>Parsing feature table<commit_after>import os
import sys

import re


def roles_of_function(role):
    """
    Separate a function into a set of roles.

    In the SEED a function associated with a PEG can have more than one role. This is denoted by joining roles
    with either ' / ' (a multi-domain, multifunctional gene), ' ; ' (an ambiguous function), or ' @ '
    (a Single domain playing multiple roles). For more information see
    http://www.nmpdr.org/FIG/Html/SEED_functions.html

    This method just takes a function and returns a set of the role(s). If there is a single role, it is still a set.

    :param role: The functional role
    :type role: str
    :return: A set of the roles
    :rtype: set
    """

    # remove flanking ", comments from functions, and split multiple functions
    func = re.sub('^"|"$|\s+[#!]\s.*$', '', role)
    return set(re.split('\s*;\s+|\s+[;/@]\s+', func))


def read_features_file(features_file, verbose=False):
    """
    Read a PATRIC features file and return a set of the roles
    :param features_file: The features file to read
    :param verbose: more output
    :return: a set of the foles
    :rtype: set[str]
    """
    if not os.path.exists(features_file):
        raise IOError(f"ERROR: {features_file} does not exist")
    roles = set()
    with open(features_file, 'r') as f:
        for li in f:
            p = li.split("\t")
            # at the moment we ignore non coding sequence features!
            if p[2] != 'CDS':
                continue
            if len(p) != 6:
                raise ValueError(f"{features_file} has {len(p)} columns, we were expecting 5 columns")
            for ro in roles_of_function(p[3]):
                roles.add(ro)
    return roles


def read_functional_roles(functional_roles_file, verbose=False):
    """
    Read a functional roles file and return a list of roles.
    :param functional_roles_file: the file with the roles
    :param verbose: more output
    :return:
    """
    if not os.path.exists(functional_roles_file):
        raise IOError(f"ERROR: {functional_roles_file} does not exist")
    roles = set()
    with open(functional_roles_file, 'r') as f:
        for li in f:
            for ro in roles_of_function(li):
                roles.add(ro)
    return roles


def read_downloaded_data(spreadsheet_file):
    """
    Read data downloaded from RAST as a 'spreadsheet (tab-separated text format)' and return a hash of the
    proteins and functional roles in the genome.
    :param spreadsheet_file:The file downloaded from RAST
{'fraction_non_alphanumeric': 0.09933774834437085, 'fraction_numerical': 0.0014047762392133253, 'mean_word_length': 2.7586726998491704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3262332', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1473, 'n_words': 562}