<commit_before>#-*- coding: utf8
'''
For a given annotation table in the database, a smooth function and 
a lambda value; this will compute the probability of the item on a 
given tag and insert it on a Mongo database.
'''
from __future__ import division, print_function

__authors__ = ['<NAME> - flaviovdf <at> gmail <dot-no-spam> com']
__date__ = '01/11/2011'

try:
    from cy_tagassess.probability_estimates import SmoothEstimator
    from cy_tagassess.value_calculator import ValueCalculator
except ImportError:
    print('!!! UNABLE TO IMPORT CYTHON MODULES ''')
    from tagassess.probability_estimates import SmoothEstimator
    from tagassess.value_calculator import ValueCalculator

from collections import Counter
from pymongo import ASCENDING
from pymongo import Connection

from tagassess.dao.mongodb.annotations import AnnotReader

import argparse
import numpy as np
import sys
import traceback

def main(database, table, smooth_func, lambda_, db_name, tname, min_tag_freq):
    
    with AnnotReader(database) as reader:
        reader.change_table(table)
        
        #Builds value calculator
        estimator = SmoothEstimator(smooth_func, lambda_, reader.iterate())
        calculator = ValueCalculator(estimator, None)
        
        #Determine tags which will be considered
        tags_to_consider = []
        if min_tag_freq < 0: #All tags
            tags_to_consider = range(estimator.num_tags())
        else:
            counter = Counter(annot['tag'] for annot in reader.iterate())
            for tag, pop in counter.iteritems():
                if pop >= min_tag_freq:
                    tags_to_consider.append(tag)
                    
        #Dumps probabilities
        connection = None
        database = None
        try:
            connection = Connection()
            database = connection[db_name]
            
            if tname in database.collection_names():
                print('Table already exists', file=sys.stderr)
                return 2
            
            table = database[tname]
            table.ensure_index([('tag', ASCENDING), ('item', ASCENDING)])
            
            gamma_items = np.arange(estimator.num_items())
            for tag in tags_to_consider:
                v_prob_it = calculator.rnorm_prob_items_given_tag(tag, 
                                                                  gamma_items)
                for item in gamma_items:
                    prob = v_prob_it[item]
                    table.insert({'tag':tag, 'item':item, 'prob_it':prob})
                
        finally:
            if connection:<commit_msg>Mongo has problems with numpy<commit_after>#-*- coding: utf8
'''
For a given annotation table in the database, a smooth function and 
a lambda value; this will compute the probability of the item on a 
given tag and insert it on a Mongo database.
'''
from __future__ import division, print_function

__authors__ = ['<NAME> - flaviovdf <at> gmail <dot-no-spam> com']
__date__ = '01/11/2011'

try:
    from cy_tagassess.probability_estimates import SmoothEstimator
    from cy_tagassess.value_calculator import ValueCalculator
except ImportError:
    print('!!! UNABLE TO IMPORT CYTHON MODULES ''')
    from tagassess.probability_estimates import SmoothEstimator
    from tagassess.value_calculator import ValueCalculator

from collections import Counter
from pymongo import ASCENDING
from pymongo import Connection

from tagassess.dao.mongodb.annotations import AnnotReader

import argparse
import numpy as np
import sys
import traceback

def main(database, table, smooth_func, lambda_, db_name, tname, min_tag_freq):
    
    with AnnotReader(database) as reader:
        reader.change_table(table)
        
        #Builds value calculator
        estimator = SmoothEstimator(smooth_func, lambda_, reader.iterate())
        calculator = ValueCalculator(estimator, None)
        
        #Determine tags which will be considered
        tags_to_consider = []
        if min_tag_freq < 0: #All tags
            tags_to_consider = range(estimator.num_tags())
        else:
            counter = Counter(annot['tag'] for annot in reader.iterate())
            for tag, pop in counter.iteritems():
                if pop >= min_tag_freq:
                    tags_to_consider.append(tag)
                    
        #Dumps probabilities
        connection = None
        database = None
        try:
            connection = Connection()
            database = connection[db_name]
            
            if tname in database.collection_names():
                print('Table already exists', file=sys.stderr)
                return 2
            
            table = database[tname]
            table.ensure_index([('tag', ASCENDING), ('item', ASCENDING)])
            
            for tag in tags_to_consider:
                v_prob_it = calculator.rnorm_prob_items_given_tag(tag)
                for item in xrange(v_prob_it):
                    prob = float(v_prob_it[item])
                    table.insert({'tag':tag, 'item':item, 'prob_it':prob})
                
        finally:
            if connection:
{'fraction_non_alphanumeric': 0.07443491816056118, 'fraction_numerical': 0.004286827747466875, 'mean_word_length': 2.261118170266836, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6506128', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1372, 'n_words': 406}