<commit_before># -*- coding: utf-8 -*-
'''
    Mango system periodic tools.
'''

# This file is part of mango.

# Dist4ributed under the terms of the last AGPL License. 
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = '<NAME>'


import logging

import motor

from contextlib import contextmanager
from tornado import gen

# from mango.messages import accounts

from bson import objectid


'''
    SELECT
        DISTINCT ON (uniqueid) uniqueid,
        calldate,
        src,
        dst,
        dcontext,
        channel,
        dstchannel,
        lastapp,
        lastdata,
        duration,
        billsec,
        disposition,
        checked

    FROM cdr

    WHERE src = '{account}'

    ORDER BY uniqueid DESC

    limit 10;
'''


@gen.coroutine
def get_usernames(db):
    '''
        Get all the username accounts
    '''
    usernames = []
    try:
        query = db.accounts.find(
            {}, 
            {'account':1, 'uuid':1, '_id':0}
        )
        while (yield query.fetch_next):
            account = query.next_object()<commit_msg>periodic get raw records stuff<commit_after># -*- coding: utf-8 -*-
'''
    Mango system periodic tools.
'''

# This file is part of mango.

# Dist4ributed under the terms of the last AGPL License. 
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = '<NAME>'


import logging

import motor
import queries

from contextlib import contextmanager
from tornado import gen

# from mango.messages import accounts

from bson import objectid


@gen.coroutine
def get_raw_records(sql, query_limit):
    '''
        Get RAW records
    '''
    try:
        # Get SQL database from mango settings
        query = '''
            SELECT
                DISTINCT ON (uniqueid) uniqueid,
                calldate,
                src,
                dst,
                dcontext,
                channel,
                dstchannel,
                lastapp,
                lastdata,
                duration,
                billsec,
                disposition,
                checked
            
            FROM cdr

            ORDER BY uniqueid DESC

            limit {0};
        '''.format(
            query_limit
        )
        result = yield sql.query(query)

        if result:
            message = {'ack': True}
        else:
            message = {'ack': False}

        result.free()

        logging.warning('get raw records spawned on PostgreSQL {0}'.format(message))
        
    except Exception, e:
        logging.exception(e)
        raise e

    raise gen.Return(message)

@gen.coroutine
def get_usernames(db):
    '''
        Get all the username accounts
    '''
    usernames = []
    try:
        query = db.accounts.find(
            {}, 
            {'account':1, 'uuid':1, '_id':0}
        )
        while (yield query.fetch_next):
            account = query.next_object()
{'fraction_non_alphanumeric': 0.0881057268722467, 'fraction_numerical': 0.004744154523890207, 'mean_word_length': 1.6546762589928057, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515525', 'n_tokens_mistral': 848, 'n_tokens_neox': 806, 'n_words': 259}