<commit_before>from datanator.util import mongo_util
import time

class DataQuery(mongo_util.MongoUtil):

    def __init__(self, cache_dirname=None, MongoDB=None, replicaSet= None, db=None,
                collection_str=None, verbose=False, max_entries=float('inf')):
        self.collection_str = collection_str
        super(DataQuery, self).__init__(cache_dirname=cache_dirname, MongoDB=MongoDB, replicaSet=replicaSet, 
                                    db=db, verbose=verbose, max_entries=max_entries)
        self.client_obj, self.db_obj, self.collection_obj = self.con_db(self.collection_str)

    '''TODO:1. Make query language more user friendly
            2. Full text search for all key:value pairs
    '''


    def find_reaction_participants(self, kinlaw_id):
        ''' Find the reaction participants defined in sabio_rk using kinetic law id
            Args:
                kinlaw_id: integer type
            Return:
                substrates: list of substrates
                products: list of products
        '''
        _, _, collection = self.con_db('sabio_rk')
        query = {'kinlaw_id': kinlaw_id}
        doc = collection.find_one(query)
        doc.pop('_id', None)
        doc_flat = self.flatten_json(doc)

        substrates = []
        products = []
        substrate_identifier = ['reaction_participant', 'substrate', 'name']
        product_identifier = ['reaction_participant', 'product', 'name']

        for k, v in doc_flat.items():
            if all(x in k for x in substrate_identifier) and 'compartment' not in k:
                substrates.append(v)
            elif all(x in k for x in product_identifier) and 'compartment' not in k:
                products.append(v)
            else:
                continue

        return (substrates, products)

    def find_text(self, v, collection=None):
        ''' Find documents containing string v
            v cannot be part of a word (e.g. 'wor' of 'word')
            v needs to be in a previously indexed field
            Args:
                v: value to be matched
        '''
        if collection == None:
            return self.collection.find({'$text': {'$search': v}},
                                        { score: { $meta: "textScore" } }).sort( { score: { $meta: "textScore" } } )
        else:
            col_obj = self.con_db(collection)
            return col_obj.find({'$text': {'$search': v}},
                                { score: { $meta: "textScore" } }).sort( { score: { $meta: "textScore" } } )

    def doc_feeder(self,collection_str=None, sym_link = False, step=1000, 
        s=None, e=None, inbatch=False, query=None, batch_callback=None, projection=None):
        '''A iterator for returning docs in a collection, with batch query.
           additional filter query can be passed via "query", e.g.,
           doc_feeder(collection_str, query={'taxid': {'$in': [9606, 10090, 10116]}})
           batch_callback is a callback function as fn(cnt, t), called after every batch
           fields is optional parameter passed to find to restrict fields to return.
        '''
        collection = self.fill_db(collection_str, sym_link=sym_link)
        cur = collection.find(query, no_cursor_timeout=False, projection=projection)
        n = cur.count()
        s = s or 0
        e = e or n
        if self.verbose:
            print('Retrieving %d documents from collection "%s".' %
                  (n, self.collection_str))
        t0 = time.time()
        if inbatch:
            doc_li = []
        cnt = 0
        t1 = time.time()
        try:
            if s:<commit_msg>start to fill metabolites meta collection<commit_after>from datanator.util import mongo_util
import time

class DataQuery(mongo_util.MongoUtil):

    def __init__(self, cache_dirname=None, MongoDB=None, replicaSet= None, db=None,
                collection_str=None, verbose=False, max_entries=float('inf')):
        self.collection_str = collection_str
        super(DataQuery, self).__init__(cache_dirname=cache_dirname, MongoDB=MongoDB, replicaSet=replicaSet, 
                                    db=db, verbose=verbose, max_entries=max_entries)
        self.client_obj, self.db_obj, self.collection_obj = self.con_db(self.collection_str)

    '''TODO:1. Make query language more user friendly
            2. Full text search for all key:value pairs
    '''


    def find_reaction_participants(self, kinlaw_id):
        ''' Find the reaction participants defined in sabio_rk using kinetic law id
            Args:
                kinlaw_id: integer type
            Return:
                substrates: list of substrates
                products: list of products
        '''
        _, _, collection = self.con_db('sabio_rk')
        query = {'kinlaw_id': kinlaw_id}
        doc = collection.find_one(query)
        doc.pop('_id', None)
        doc_flat = self.flatten_json(doc)

        substrates = []
        products = []
        substrate_identifier = ['reaction_participant', 'substrate', 'name']
        product_identifier = ['reaction_participant', 'product', 'name']

        for k, v in doc_flat.items():
            if all(x in k for x in substrate_identifier) and 'compartment' not in k:
                substrates.append(v)
            elif all(x in k for x in product_identifier) and 'compartment' not in k:
                products.append(v)
            else:
                continue

        return (substrates, products)

    def find_text(self, v, collection=None):
        ''' Find documents containing string v
            v cannot be part of a word (e.g. 'wor' of 'word')
            v needs to be in a previously indexed field
            Args:
                v: value to be matched
        '''
        if collection == None:
            return self.collection.find({'$text': {'$search': v}},
                                        { 'score': { '$meta': "textScore" } }).sort( { 'score': { '$meta': "textScore" } } )
        else:
            col_obj = self.con_db(collection)
            return col_obj.find({'$text': {'$search': v}},
                                { 'score': { '$meta': "textScore" } }).sort( { 'score': { '$meta': "textScore" } } )

    def doc_feeder(self,collection_str=None, sym_link = False, step=1000, 
        s=None, e=None, inbatch=False, query=None, batch_callback=None, projection=None):
        '''A iterator for returning docs in a collection, with batch query.
           additional filter query can be passed via "query", e.g.,
           doc_feeder(collection_str, query={'taxid': {'$in': [9606, 10090, 10116]}})
           batch_callback is a callback function as fn(cnt, t), called after every batch
           fields is optional parameter passed to find to restrict fields to return.
        '''
        collection = self.fill_db(collection_str, sym_link=sym_link)
        cur = collection.find(query, no_cursor_timeout=False, projection=projection)
        n = cur.count()
        s = s or 0
        e = e or n
        if self.verbose:
            print('Retrieving %d documents from collection "%s".' %
                  (n, self.collection_str))
        t0 = time.time()
        if inbatch:
            doc_li = []
        cnt = 0
        t1 = time.time()
        try:
            if s:
{'fraction_non_alphanumeric': 0.10643564356435643, 'fraction_numerical': 0.006600660066006601, 'mean_word_length': 2.105465414175918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5260045', 'n_tokens_mistral': 2112, 'n_tokens_neox': 1999, 'n_words': 596}