<commit_before>'''Parse SabioRk json files into MongoDB documents
    (json files acquired by running sqlite_to_json.py)
'''

import json
import pymongo
from pymongo import MongoClient
from pathlib import Path
import re
import os

class SabioRkNoSQL():

    def __init__(self, directory, db, MongoDB, max_entries=float('inf')):
        '''
                Attributes:
                        directory: temporary os directory
                        db: mongodb database name
                        MongoDB: MongoDB server address and login e.g. 'mongodb://localhost:27017/'

        '''
        self.directory = directory
        self.MongoDB = MongoDB
        self.db = db
        self.collection = 'sabio_rk'
        self.max_entries = max_entries

    # make connections wth mongoDB
    def con_db(self):
        try:
            client = MongoClient(self.MongoDB, 400)  # 400ms max timeout
            client.server_info()
            db = client[self.db]
            collection = db[self.collection]
            return collection
        except pymongo.errors.ConnectionFailure:
            print('Server not available')

    # load json files
    def load_json(self):
        pathlist = Path(self.directory).glob('**/*.json')
        file_names = []
        file_dict = {}
        for path in pathlist:
            path_in_str = str(path)
            name = re.findall(r'[^\/]+(?=\.json$)', path_in_str)[0]
            file_names.append(name)
            with open(path_in_str) as f:
                file_dict[name] = json.load(f)
        return (file_names, file_dict)

    '''loads dictionaries as documents (1 kinetic_law per document)
        Attributes:

            file_names: list of name of files
            file_dict: dictionaries of info for each file e.g.
                        {'entry': {...}, 'kinetic_law': {...},  }
            returns a json document in the format:
            {
                'kinlaw_id': 1
                'compartment:' []
                ...
            }
    '''

    def make_doc(self, file_names, file_dict):
        '''
        Start iteration with kinetic_law

        '''
        os.makedirs(os.path.dirname(self.directory + 'docs/'), exist_ok=True)
        null = None
        for key, value in file_dict.items():
            # might not be a good idea
            globals()[key+ '_list'] = value
            
        # list of entrie ids that have synonyms
        has_synonym = list(item['entry__id'] for item in entry_synonym_list)
        # list of entrie ids that have compound structure information
        has_structure = list(item['compound__id'] for item in compound_compound_structure_list)

        for i in range(min(len(kinetic_law_list), self.max_entries)):
            cur_kinlaw_dict = kinetic_law_list[i]
            kinlaw_id = next(item['id'] for item in entry_list if item['_id'] == cur_kinlaw_dict['_id'])
            json_name = self.directory+'docs/'+'kinlaw_id_' + str(kinlaw_id) + '.json'
            sabio_doc = {}
            sabio_doc['kinlaw_id'] = kinlaw_id          
            sabio_doc['resource'] = [] 
            sabio_doc['enzymes'] = [{'enzyme': []},{'compartment': []},{'subunit': []}] 
            sabio_doc['parameter'] = []
            sabio_doc['reaction_participant'] = [{'substrate': []}, {'product': []}, {'modifier': []}]

            # every kinlaw entry has no more than one enzyme
            enzyme_id = cur_kinlaw_dict['enzyme_id']
            if enzyme_id != null:
                cur_enzyme_dict = next(item for item in enzyme_list if item['_id'] == enzyme_id )
                cur_enzyme_entry_dict = next(item for item in entry_list if item['_id'] == enzyme_id )
            else:
                sabio_doc['enzymes'][0]['enzyme'].append({
                'molecular_weight': None, 
                'id': None,
                'name': None,
                'type': None,
                'synonym': None,
                'created': None,
                'modified': None})   

            cur_enzyme_subunit_list = list(item for item in enzyme_subunit_list if item['enzyme_id'] == enzyme_id) # enzyme_id == entry_id
            enzyme_compartment_id = cur_kinlaw_dict['enzyme_compartment_id']
            cur_compartment_list = list(item for item in entry_list if item['_id'] == enzyme_compartment_id) # enzyme_compartment_id = compartment_id = entry_id

            enzyme_synonym = []
            if enzyme_id in has_synonym:
                synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == enzyme_id)
                for synonym_id in synonym_id_list:
                    enzyme_synonym.append(synonym_list[synonym_id - 1]['name']) 
                

            sabio_doc['enzymes'][0]['enzyme'].append({
            'molecular_weight': cur_enzyme_dict['molecular_weight'], 
            'id': cur_enzyme_entry_dict['id'],
            'name': cur_enzyme_entry_dict['name'],
            'type': cur_kinlaw_dict['enzyme_type'],
            'synonym': enzyme_synonym,
            'created': cur_enzyme_entry_dict['created'],
            'modified': cur_enzyme_entry_dict['modified']})

            if len(cur_enzyme_subunit_list) == 0:
                sabio_doc['enzymes'][2]['subunit'].append({
                'id': None,
                'name': None,
                'coefficient': None,
                'sequence': None,
                'molecular_weight': None,
                'created': None,
                'modified': None
                })   
            else:
                for j in range(len(cur_enzyme_subunit_list)):
                    cur_enzyme_subunit_dict = cur_enzyme_subunit_list[j]
                    entry_id = cur_enzyme_subunit_dict['_id']
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == entry_id )
                    sabio_doc['enzymes'][2]['subunit'].append({
                    'id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'coefficient': cur_enzyme_subunit_dict['coefficient'],
                    'sequence': cur_enzyme_subunit_dict['sequence'],
                    'molecular_weight': cur_enzyme_subunit_dict['molecular_weight'],
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    })

            if len(cur_compartment_list) == 0:
                sabio_doc['enzymes'][1]['compartment'].append({
                'id': None,
                'name': None,
                'created': None,
                'modified': None
                })
            else:
                for j in range(len(cur_compartment_list)):
                    cur_compartment_dict = cur_compartment_list[j]
                    sabio_doc['enzymes'][1]['compartment'].append({
                    'id': cur_compartment_dict['id'],
                    'name': cur_compartment_dict['name'],
                    'created': cur_compartment_dict['created'],
                    'modified': cur_compartment_dict['modified']
                    })

            # kinetic_law_resource many-to-one
            cur_kinlaw_resource_dict = next(item for item in kinetic_law_resource_list if item['kinetic_law__id'] == cur_kinlaw_dict['_id'])
            # entry_resource: same entry id can have multiple resource_id
            cur_entry_resource_list = list(item for item in entry_resource_list if item['entry__id'] == cur_kinlaw_dict['_id'])

            if not cur_kinlaw_resource_dict:
                sabio_doc['resource'].append({
                'namespace': None,
                'id': None
                })
            else:
                resource_id = cur_kinlaw_resource_dict['resource__id']
                cur_resource_dict = next(item for item in resource_list if item['_id'] == resource_id)
                sabio_doc['resource'].append({
                'namespace': cur_resource_dict['namespace'],
                'id': cur_resource_dict['id']
                })

            if len(cur_entry_resource_list) != 0:
                for j in range(len(cur_entry_resource_list)):
                    resource__id = cur_entry_resource_list[j]['resource__id']
                    cur_resource_dict = next(item for item in resource_list if item['_id'] == resource__id)
                    sabio_doc['resource'].append({
                        'namespace': cur_resource_dict['namespace'],
                        'id': cur_resource_dict['id']
                        })

            cur_reaction_reactant_list = list(item for item in reaction_participant_list if item['reactant_kinetic_law_id'] == cur_kinlaw_dict['_id'])
            cur_reaction_product_list = list(item for item in reaction_participant_list if item['product_kinetic_law_id'] == cur_kinlaw_dict['_id'])
            cur_reaction_modifier_list = list(item for item in reaction_participant_list if item['modifier_kinetic_law_id'] == cur_kinlaw_dict['_id'])

            for j in range(len(cur_reaction_reactant_list)):
                cur_reactant_dict = cur_reaction_reactant_list[j]
                cur_reactant_compound_entry_id = cur_reactant_dict['compound_id']
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []
                
                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})
                
                cur_reactant_compartment_id = cur_reactant_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][0]['substrate'].append({
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    })

            for j in range(len(cur_reaction_product_list)):
                cur_reactant_dict = cur_reaction_product_list[j]
                cur_reactant_compound_entry_id = cur_reactant_dict['compound_id']
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []

                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})

                cur_reactant_compartment_id = cur_reactant_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][1]['product'].append({
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],                    
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    })

            for j in range(len(cur_reaction_modifier_list)):
                cur_modifier_dict = cur_reaction_modifier_list[j]
                cur_reactant_compound_entry_id = cur_modifier_dict['compound_id']
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []

                cur_reactant_compartment_id = cur_modifier_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][2]['modifier'].append({
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],                    
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    })               

            cur_parameter_list = list(item for item in parameter_list if item['kinetic_law_id'] == cur_kinlaw_dict['_id'])
            if len(cur_parameter_list) != 0:
                for j in range(len(cur_parameter_list)):
                    entry__id = cur_parameter_list[j]['_id']
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == entry__id)

                    _type = cur_parameter_list[j]['type']

                    compound_id = cur_parameter_list[j]['compound_id']
                    cur_compound_list = list(item for item in compound_list if item['_id'] == compound_id)
                    if len(cur_compound_list) == 0:
                        cur_compound_dict = {}
                    else:
                        cur_compound_dict = cur_compound_list[0]<commit_msg>sabio_rk_nosql.py added kegg, chebi, pubchem, and uniprot information<commit_after>'''Parse SabioRk json files into MongoDB documents
    (json files acquired by running sqlite_to_json.py)
'''

import json
import pymongo
from pymongo import MongoClient
from pathlib import Path
import re
import os

class SabioRkNoSQL():

    def __init__(self, directory, db, MongoDB, max_entries=float('inf')):
        '''
                Attributes:
                        directory: temporary os directory
                        db: mongodb database name
                        MongoDB: MongoDB server address and login e.g. 'mongodb://localhost:27017/'

        '''
        self.directory = directory
        self.MongoDB = MongoDB
        self.db = db
        self.collection = 'sabio_rk'
        self.max_entries = max_entries

    # make connections wth mongoDB
    def con_db(self):
        try:
            client = MongoClient(self.MongoDB, 400)  # 400ms max timeout
            client.server_info()
            db = client[self.db]
            collection = db[self.collection]
            return collection
        except pymongo.errors.ConnectionFailure:
            print('Server not available')

    # load json files
    def load_json(self):
        pathlist = Path(self.directory).glob('**/*.json')
        file_names = []
        file_dict = {}
        for path in pathlist:
            path_in_str = str(path)
            name = re.findall(r'[^\/]+(?=\.json$)', path_in_str)[0]
            file_names.append(name)
            with open(path_in_str) as f:
                file_dict[name] = json.load(f)
        return (file_names, file_dict)

    '''loads dictionaries as documents (1 kinetic_law per document)
        Attributes:

            file_names: list of name of files
            file_dict: dictionaries of info for each file e.g.
                        {'entry': {...}, 'kinetic_law': {...},  }
            returns a json document in the format:
            {
                'kinlaw_id': 1
                'compartment:' []
                ...
            }
    '''

    def make_doc(self, file_names, file_dict):
        '''
        Start iteration with kinetic_law

        '''
        os.makedirs(os.path.dirname(self.directory + 'docs/'), exist_ok=True)
        null = None
        for key, value in file_dict.items():
            # might not be a good idea
            globals()[key+ '_list'] = value
            
        # list of entrie ids that have synonyms
        has_synonym = list(item['entry__id'] for item in entry_synonym_list)
        # list of entrie ids that have compound structure information
        has_structure = list(item['compound__id'] for item in compound_compound_structure_list)

        for i in range(min(len(kinetic_law_list), self.max_entries)):
            cur_kinlaw_dict = kinetic_law_list[i]
            kinlaw_id = next(item['id'] for item in entry_list if item['_id'] == cur_kinlaw_dict['_id'])
            json_name = self.directory+'docs/'+'kinlaw_id_' + str(kinlaw_id) + '.json'
            sabio_doc = {}
            sabio_doc['kinlaw_id'] = kinlaw_id          
            sabio_doc['resource'] = [] 
            sabio_doc['enzymes'] = [{'enzyme': []},{'compartment': []},{'subunit': []}] 
            sabio_doc['parameter'] = []
            sabio_doc['reaction_participant'] = [{'substrate': []}, {'product': []}, {'modifier': []}]

            # every kinlaw entry has no more than one enzyme
            enzyme_id = cur_kinlaw_dict['enzyme_id']
            if enzyme_id != null:
                cur_enzyme_dict = next(item for item in enzyme_list if item['_id'] == enzyme_id )
                cur_enzyme_entry_dict = next(item for item in entry_list if item['_id'] == enzyme_id )
            else:
                sabio_doc['enzymes'][0]['enzyme'].append({
                'molecular_weight': None, 
                'id': None,
                'name': None,
                'type': None,
                'synonym': None,
                'created': None,
                'modified': None})   

            cur_enzyme_subunit_list = list(item for item in enzyme_subunit_list if item['enzyme_id'] == enzyme_id) # enzyme_id == entry_id
            enzyme_compartment_id = cur_kinlaw_dict['enzyme_compartment_id']
            cur_compartment_list = list(item for item in entry_list if item['_id'] == enzyme_compartment_id) # enzyme_compartment_id = compartment_id = entry_id

            enzyme_synonym = []
            if enzyme_id in has_synonym:
                synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == enzyme_id)
                for synonym_id in synonym_id_list:
                    enzyme_synonym.append(synonym_list[synonym_id - 1]['name']) 
                

            sabio_doc['enzymes'][0]['enzyme'].append({
            'molecular_weight': cur_enzyme_dict['molecular_weight'], 
            'id': cur_enzyme_entry_dict['id'],
            'name': cur_enzyme_entry_dict['name'],
            'type': cur_kinlaw_dict['enzyme_type'],
            'synonym': enzyme_synonym,
            'created': cur_enzyme_entry_dict['created'],
            'modified': cur_enzyme_entry_dict['modified']})

            if len(cur_enzyme_subunit_list) == 0:
                sabio_doc['enzymes'][2]['subunit'].append({
                'id': None,
                'name': None,
                'uniprot_id': None,
                'coefficient': None,
                'sequence': None,
                'molecular_weight': None,
                'created': None,
                'modified': None
                })   
            else:
                for j in range(len(cur_enzyme_subunit_list)):
                    cur_enzyme_subunit_dict = cur_enzyme_subunit_list[j]
                    entry_id = cur_enzyme_subunit_dict['_id']
                    resource_id = next(item['resource__id'] for item in entry_resource_list if item['entry__id'] == entry_id)
                    uniprot_id = next(item['id'] for item in resource_list if item['_id'] == resource_id)
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == entry_id )

                    sabio_doc['enzymes'][2]['subunit'].append({
                    'id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'uniprot_id': uniprot_id,
                    'coefficient': cur_enzyme_subunit_dict['coefficient'],
                    'sequence': cur_enzyme_subunit_dict['sequence'],
                    'molecular_weight': cur_enzyme_subunit_dict['molecular_weight'],
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    })

            if len(cur_compartment_list) == 0:
                sabio_doc['enzymes'][1]['compartment'].append({
                'id': None,
                'name': None,
                'created': None,
                'modified': None
                })
            else:
                for j in range(len(cur_compartment_list)):
                    cur_compartment_dict = cur_compartment_list[j]
                    sabio_doc['enzymes'][1]['compartment'].append({
                    'id': cur_compartment_dict['id'],
                    'name': cur_compartment_dict['name'],
                    'created': cur_compartment_dict['created'],
                    'modified': cur_compartment_dict['modified']
                    })

            # kinetic_law_resource many-to-one
            cur_kinlaw_resource_dict = next(item for item in kinetic_law_resource_list if item['kinetic_law__id'] == cur_kinlaw_dict['_id'])
            # entry_resource: same entry id can have multiple resource_id
            cur_entry_resource_list = list(item for item in entry_resource_list if item['entry__id'] == cur_kinlaw_dict['_id'])

            if not cur_kinlaw_resource_dict:
                sabio_doc['resource'].append({
                'namespace': None,
                'id': None
                })
            else:
                resource_id = cur_kinlaw_resource_dict['resource__id']
                cur_resource_dict = next(item for item in resource_list if item['_id'] == resource_id)
                sabio_doc['resource'].append({
                'namespace': cur_resource_dict['namespace'],
                'id': cur_resource_dict['id']
                })

            if len(cur_entry_resource_list) != 0:
                for j in range(len(cur_entry_resource_list)):
                    resource__id = cur_entry_resource_list[j]['resource__id']
                    cur_resource_dict = next(item for item in resource_list if item['_id'] == resource__id)
                    sabio_doc['resource'].append({
                        'namespace': cur_resource_dict['namespace'],
                        'id': cur_resource_dict['id']
                        })

            '''Handling reaction_participant document
                substrate, product, modifier
            '''
            cur_reaction_reactant_list = list(item for item in reaction_participant_list if item['reactant_kinetic_law_id'] == cur_kinlaw_dict['_id'])
            cur_reaction_product_list = list(item for item in reaction_participant_list if item['product_kinetic_law_id'] == cur_kinlaw_dict['_id'])
            cur_reaction_modifier_list = list(item for item in reaction_participant_list if item['modifier_kinetic_law_id'] == cur_kinlaw_dict['_id'])

            # substrates
            for j in range(len(cur_reaction_reactant_list)):
                cur_reactant_dict = cur_reaction_reactant_list[j]
                cur_reactant_compound_entry_id = cur_reactant_dict['compound_id']

                resource__id = list(item['resource__id'] for item in entry_resource_list if item['entry__id'] == cur_reactant_compound_entry_id)
                resources = {resource_list[x-1]['namespace']: resource_list[x-1]['id'] for x in resource__id}
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []
                
                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})
                
                cur_reactant_compartment_id = cur_reactant_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][0]['substrate'].append({ **{
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    }, **resources})

            # product
            for j in range(len(cur_reaction_product_list)):
                cur_reactant_dict = cur_reaction_product_list[j]
                cur_reactant_compound_entry_id = cur_reactant_dict['compound_id']

                resource__id = list(item['resource__id'] for item in entry_resource_list if item['entry__id'] == cur_reactant_compound_entry_id)
                resources = {resource_list[x-1]['namespace']: resource_list[x-1]['id'] for x in resource__id}
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []

                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})

                cur_reactant_compartment_id = cur_reactant_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][1]['product'].append({ **{
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],                    
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    }, **resources})

            # modifier
            for j in range(len(cur_reaction_modifier_list)):
                cur_modifier_dict = cur_reaction_modifier_list[j]
                cur_reactant_compound_entry_id = cur_modifier_dict['compound_id']

                resource__id = list(item['resource__id'] for item in entry_resource_list if item['entry__id'] == cur_reactant_compound_entry_id)
                resources = {resource_list[x-1]['namespace']: resource_list[x-1]['id'] for x in resource__id}
                # compound synonym if there is any
                cur_reactant_synonym_id_list = list(item['synonym__id'] for item in entry_synonym_list if item['entry__id'] == cur_reactant_compound_entry_id)
                if len(cur_reactant_synonym_id_list) != 0:
                    reactant_synonyms = [synonym_list[x-1]['name'] for x in cur_reactant_synonym_id_list]
                else:
                    reactant_synonyms = []

                cur_reactant_compartment_id = cur_modifier_dict.get('compartment_id')
                if cur_reactant_compartment_id != None:
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compartment_id)
                    compartment_name = cur_entry_dict['name']
                    compartment_created = cur_entry_dict['created']
                    compartment_modified = cur_entry_dict['modified']
                    reactant_compartment = {'name':compartment_name, 
                    'created': compartment_created, 
                    'modified':compartment_modified}
                else:
                    reactant_compartment = {}

                # compound structure if there is any
                cur_compound_structure = []
                if cur_reactant_compound_entry_id in has_structure:
                    structure_id_list = list(item['compound_structure__id'] for item in compound_compound_structure_list if item['compound__id'] == cur_reactant_compound_entry_id)
                    for structure_id in structure_id_list:
                        cur_compound_structure.append({'value': compound_structure_list[structure_id - 1]['value'],
                            'format': compound_structure_list[structure_id - 1]['format'],
                            'inchi': compound_structure_list[structure_id - 1]['_value_inchi'],
                            'inchi_connectivity': compound_structure_list[structure_id - 1]['_value_inchi_formula_connectivity']})

                cur_reactant_coefficient = cur_reactant_dict['coefficient']
                cur_reactant_type = cur_reactant_dict['type']
                cur_entry_dict = next(item for item in entry_list if item['_id'] == cur_reactant_compound_entry_id)
                sabio_doc['reaction_participant'][2]['modifier'].append({ **{
                    'sabio_compound_id': cur_entry_dict['id'],
                    'name': cur_entry_dict['name'],
                    'synonym': reactant_synonyms,
                    'structure': cur_compound_structure,
                    'compartment': reactant_compartment,
                    'coefficient': cur_reactant_dict['coefficient'],
                    'type': cur_reactant_dict['type'],                    
                    'created': cur_entry_dict['created'],
                    'modified': cur_entry_dict['modified']
                    }, **resources})               

            '''Handling parameter
            '''
            cur_parameter_list = list(item for item in parameter_list if item['kinetic_law_id'] == cur_kinlaw_dict['_id'])
            if len(cur_parameter_list) != 0:
                for j in range(len(cur_parameter_list)):
                    entry__id = cur_parameter_list[j]['_id']
                    cur_entry_dict = next(item for item in entry_list if item['_id'] == entry__id)

                    _type = cur_parameter_list[j]['type']

                    compound_id = cur_parameter_list[j]['compound_id']
                    cur_compound_list = list(item for item in compound_list if item['_id'] == compound_id)
                    if len(cur_compound_list) == 0:
                        cur_compound_dict = {}
                    else:
                        cur_compound_dict = cur_compound_list[0]
{'fraction_non_alphanumeric': 0.09650452088700308, 'fraction_numerical': 0.002576147901197151, 'mean_word_length': 1.9946301618514597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964092', 'n_tokens_mistral': 11407, 'n_tokens_neox': 10680, 'n_words': 2018}