<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os
import argparse
import pickle

###########
# Methods #
###########
def retrieve_gene_cog_mapping_seq_length(cog_info_filepath, seq_length_filepath, 
    gene_cog_mapping_filepath, protein_organism_mapping_filepath):
    cog_info_file = open(cog_info_filepath, 'r')
    
    cogs = {}
    gene_length_file = open(seq_length_filepath,'w')
    prot_organisms = {}
    Organisms_cogs = {}
    for line in cog_info_file.readlines():
        split_line = line[:-2].split(',')
        prot_id = split_line[2]
        organism = split_line[1]
        prot_length = split_line[3]
        cog_id = split_line[6]

        if not prot_organisms.has_key(prot_id):
            prot_organisms[prot_id] = organism
        elif prot_organisms[prot_id] != organism : 
            string = os.path.basename(__file__) + ' -- ' + inspect.stack()[1][3]
            string += ": " + prot_id 
            string += " already register with a different organism"
            raise ValueError(string)

        string = organism.lower() + ':' + prot_id + '\t' + prot_length + '\n'
        gene_length_file.write(string)

        cogs.setdefault(cog_id, []).append(organism + '#' + prot_id)

        Organisms_cogs.setdefault(organism, []).append(cog_id)
    gene_length_file.close()
    cog_info_file.close()

    gene_to_cog_file = open(gene_cog_mapping_filepath,'w')
    for cog in cogs:
        gene_to_cog_file.write(cog)
        for prot_id in cogs[cog]:
            gene_to_cog_file.write('\t' + prot_id.upper())
        gene_to_cog_file.write('\n') 
    gene_to_cog_file.close()

    protein_organism_mapping_file = open(protein_organism_mapping_filepath,'w')
    pickle.dump(prot_organisms,protein_organism_mapping_file)
    protein_organism_mapping_file.close()

    return Organisms_cogs

def retrieve_kegg_cog_mapping(filepath):
    kegg_cog_mapping_file = open(filepath,'r')
    kegg_cog_mapping_file_content = kegg_cog_mapping_file.readlines()
    kegg_cog_mapping_file.close()

    kegg_cog_mapping = {}
    for line in kegg_cog_mapping_file_content:
        split_line = line[:-1].split('\t')
        ko = split_line[0]
        cog = split_line[1:]

        kegg_cog = kegg_cog_mapping.setdefault(ko, [])
        kegg_cog += cog

    compt = 0
    for k in kegg_cog_mapping.keys():
        if len(kegg_cog_mapping[k]) > 1:
            compt += 1 
    print "\t\t", compt,"kegg categories with multiple cog"

    return kegg_cog_mapping

def test_kegg_presence(k, K_categories, kegg_cog_mapping, K_not_found):
    if k not in K_categories:
        K_categories.append(k)
    if not kegg_cog_mapping.has_key(k):
        if k not in K_not_found:
            K_not_found.append(k)
        return False
    else : 
        return True

def generation_cog_pathway_mapping(kegg_pathway_mapping_filepath, 
    cog_pathway_mapping_filepath, kegg_cog_mapping):
    kegg_pathway_mapping_file = open(kegg_pathway_mapping_filepath,'r')
    cog_pathway_mapping_file = open(cog_pathway_mapping_filepath,'w')
    cog_ko_mapping = {}
    for line in  kegg_pathway_mapping_file.readlines():
        split_line = line[:-1].split('\t')
        ko = split_line[0]
        ks = split_line[1:]
        module_string = ''
        for k in ks :
            if test_kegg_presence(k, K_categories,kegg_cog_mapping,K_not_found):
                module_string += '\t' + kegg_cog_mapping[k][0]
                for cog in kegg_cog_mapping[k]:
                    cog_ko_mapping.setdefault(cog, []).append(ko)
        if module_string != '':
            cog_pathway_mapping_file.write(ko + module_string + '\n')
    cog_pathway_mapping_file.close()
    kegg_pathway_mapping_file.close()
    return cog_ko_mapping

def check_transformation(module_string):
    new_module_string = module_string

    open_bracket = 0
    module_nb = 0
    for s in new_module_string:
        if s == '(':
            open_bracket += 1
    return new_module_string

def transform_combination_from_kegg_to_cog(module):
    if isinstance(module,str):
        return ''
    if module._isleaf():
        k = module.m_pToken
        if test_kegg_presence(k, K_categories,kegg_cog_mapping,K_not_found):
            return kegg_cog_mapping[k][0]
        else : 
            return ''
    else :
        #s = '('
        s = ''
        #print "module", module
        module_str = str(module)
        module_num = 0
        found_module = 0
        for submodule in module.m_pToken:
            module_num += 1
            pos = module_str.find(str(submodule))
            #print 'submodule', submodule, module_str[pos-1]
            kegg_to_cog_str = transform_combination_from_kegg_to_cog(submodule)
            #print 'kegg_to_cog_str',kegg_to_cog_str
            if kegg_to_cog_str != '':
                found_module += 1
                #if s == '' :
                #    if module_num != len(module.m_pToken):
                #        s += '('
                if s != '' :
                    s += module_str[pos-1]
                s += kegg_to_cog_str
        if s != '' and found_module > 1:
            s = '(' + s + ')'
            #s += ')'

        s = check_transformation(s)
        print module, s
        return s
<commit_msg>Remove some prints<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os
import argparse
import pickle

###########
# Methods #
###########
def retrieve_gene_cog_mapping_seq_length(cog_info_filepath, seq_length_filepath, 
    gene_cog_mapping_filepath, protein_organism_mapping_filepath):
    cog_info_file = open(cog_info_filepath, 'r')
    
    cogs = {}
    gene_length_file = open(seq_length_filepath,'w')
    prot_organisms = {}
    Organisms_cogs = {}
    for line in cog_info_file.readlines():
        split_line = line[:-2].split(',')
        prot_id = split_line[2]
        organism = split_line[1]
        prot_length = split_line[3]
        cog_id = split_line[6]

        if not prot_organisms.has_key(prot_id):
            prot_organisms[prot_id] = organism
        elif prot_organisms[prot_id] != organism : 
            string = os.path.basename(__file__) + ' -- ' + inspect.stack()[1][3]
            string += ": " + prot_id 
            string += " already register with a different organism"
            raise ValueError(string)

        string = organism.lower() + ':' + prot_id + '\t' + prot_length + '\n'
        gene_length_file.write(string)

        cogs.setdefault(cog_id, []).append(organism + '#' + prot_id)

        Organisms_cogs.setdefault(organism, []).append(cog_id)
    gene_length_file.close()
    cog_info_file.close()

    gene_to_cog_file = open(gene_cog_mapping_filepath,'w')
    for cog in cogs:
        gene_to_cog_file.write(cog)
        for prot_id in cogs[cog]:
            gene_to_cog_file.write('\t' + prot_id.upper())
        gene_to_cog_file.write('\n') 
    gene_to_cog_file.close()

    protein_organism_mapping_file = open(protein_organism_mapping_filepath,'w')
    pickle.dump(prot_organisms,protein_organism_mapping_file)
    protein_organism_mapping_file.close()

    return Organisms_cogs

def retrieve_kegg_cog_mapping(filepath):
    kegg_cog_mapping_file = open(filepath,'r')
    kegg_cog_mapping_file_content = kegg_cog_mapping_file.readlines()
    kegg_cog_mapping_file.close()

    kegg_cog_mapping = {}
    for line in kegg_cog_mapping_file_content:
        split_line = line[:-1].split('\t')
        ko = split_line[0]
        cog = split_line[1:]

        kegg_cog = kegg_cog_mapping.setdefault(ko, [])
        kegg_cog += cog

    compt = 0
    for k in kegg_cog_mapping.keys():
        if len(kegg_cog_mapping[k]) > 1:
            compt += 1 
    print "\t\t", compt,"kegg categories with multiple cog"

    return kegg_cog_mapping

def test_kegg_presence(k, K_categories, kegg_cog_mapping, K_not_found):
    if k not in K_categories:
        K_categories.append(k)
    if not kegg_cog_mapping.has_key(k):
        if k not in K_not_found:
            K_not_found.append(k)
        return False
    else : 
        return True

def generation_cog_pathway_mapping(kegg_pathway_mapping_filepath, 
    cog_pathway_mapping_filepath, kegg_cog_mapping):
    kegg_pathway_mapping_file = open(kegg_pathway_mapping_filepath,'r')
    cog_pathway_mapping_file = open(cog_pathway_mapping_filepath,'w')
    cog_ko_mapping = {}
    for line in  kegg_pathway_mapping_file.readlines():
        split_line = line[:-1].split('\t')
        ko = split_line[0]
        ks = split_line[1:]
        module_string = ''
        for k in ks :
            if test_kegg_presence(k, K_categories,kegg_cog_mapping,K_not_found):
                module_string += '\t' + kegg_cog_mapping[k][0]
                for cog in kegg_cog_mapping[k]:
                    cog_ko_mapping.setdefault(cog, []).append(ko)
        if module_string != '':
            cog_pathway_mapping_file.write(ko + module_string + '\n')
    cog_pathway_mapping_file.close()
    kegg_pathway_mapping_file.close()
    return cog_ko_mapping

def check_transformation(module_string):
    new_module_string = module_string

    open_bracket = 0
    module_nb = 0
    for s in new_module_string:
        if s == '(':
            open_bracket += 1
    return new_module_string

def transform_combination_from_kegg_to_cog(module):
    if isinstance(module,str):
        return ''
    if module._isleaf():
        k = module.m_pToken
        if test_kegg_presence(k, K_categories,kegg_cog_mapping,K_not_found):
            return kegg_cog_mapping[k][0]
        else : 
            return ''
    else :
        #s = '('
        s = ''
        #print "module", module
        module_str = str(module)
        module_num = 0
        found_module = 0
        for submodule in module.m_pToken:
            module_num += 1
            pos = module_str.find(str(submodule))
            #print 'submodule', submodule, module_str[pos-1]
            kegg_to_cog_str = transform_combination_from_kegg_to_cog(submodule)
            #print 'kegg_to_cog_str',kegg_to_cog_str
            if kegg_to_cog_str != '':
                found_module += 1
                #if s == '' :
                #    if module_num != len(module.m_pToken):
                #        s += '('
                if s != '' :
                    s += module_str[pos-1]
                s += kegg_to_cog_str
        if s != '' and found_module > 1:
            s = '(' + s + ')'
            #s += ')'

        return s

{'fraction_non_alphanumeric': 0.10705735899645695, 'fraction_numerical': 0.005553959590156085, 'mean_word_length': 2.5011733154542406, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '125664', 'n_tokens_mistral': 3690, 'n_tokens_neox': 3581, 'n_words': 599}