<commit_before>import logging
import time
import sys
import math

from collections import defaultdict
from types import NoneType

from openspending import mongo
from openspending import model
from openspending.lib import util
from openspending.lib.aggregator import _aggregation_query

log = logging.getLogger(__name__)


class Cube(object):
    '''A cube that preaggregates entries.

    Cubes are tied to datasets. It can be used if it is defined
    for the dataset::

      {'name': 'exampledataset',
       'cubes': {'examplecube': {'dimensions': ['dim1', 'dim2', 'dim3']}}}

    A cube contains cells for each unique combination of cube dimension
    and measures the amount (`amount`) and the number of entries
    (`num_entries`) of all entries aggregated in the cell.

    Raises: :exc:`AssertionError` if a cube with the name is not defined for
    the dataset.
    '''

    def __init__(self, dataset, cube_name):
        """
        Arguments:
        ``dataset``
            A dict-like ``dataset`` object
        ``cube_name``
            The name of the cube

        Raises: :exc:`ValueError` if the dimensions specified for the
        cube *cube_name* are not allowed for the *dataset*. See
        :meth:`_cube_dimensions`.
        """
        self.dataset = dataset
        self.name = cube_name
        self.cube_description = self._cube_description()
        self.collection_name = 'cubes.%s.%s' % (dataset['name'], cube_name)
        self.dimensions = self._cube_dimensions()
        self.db = mongo.db
        self.simpletypes = (str, unicode, int, float, NoneType)

    def compute(self):
        """
        Create the cube. This processes all entries of the dataset,
        aggregates cells based on the dimensions of the cube and
        saves them into a mongodb collection in the cubes namespace.
        """
        log.info("Computing cube '%s'...", self.name)
        log.debug(" dataset: '%s'\n  dimensions: '%s'",
                  self.dataset['name'], ', '.join(self.dimensions))
        begin = time.time()

        # query fields: We query for all fields, but handle the date
        # have to query for 'time' if dates are involved.
        # time is a required field for entries, and some datasets
        # add a dimension for time, others don't.
        # If we specify cubes, we do it with 'year' (and maybe 'month')
        query_dimensions = set(self.dimensions)
        used_time_dimensions = query_dimensions.intersection(['year', 'month'])
        additional_dimensions = ['amount']
        if used_time_dimensions:
            query_dimensions = query_dimensions - used_time_dimensions
            additional_dimensions.append('time')
        query_dimensions = query_dimensions.union(additional_dimensions)
        cursor = _aggregation_query(self.dataset, {}, fields=list(query_dimensions))

        # remove a collection if there is one
        if self.is_computed():
            self.db.drop_collection(self.collection_name)
        collection = self.db[self.collection_name]

        for row in cursor:

            cell_id = self._cell_id_for_row(row, query_dimensions)
            cell = collection.find_one({'_id': cell_id})

            if cell is None:
                new_cell = {'_id': cell_id}
                for key in query_dimensions:
                    # handle dates especially, collect year and month
                    if key == 'time':
                        if 'year' in used_time_dimensions:
                            value = int(util.deep_get(row, 'time.from.year'))
                            new_cell['year'] = value
                        if 'month' in used_time_dimensions:
                            value = int(util.deep_get(row, 'time.from.month')[-2:])
                            new_cell['month'] = value
                        continue

                    value = util.deep_get(row, key)
                    if isinstance(value, dict):
                        from_day = util.deep_get(value, 'from.day')
                        if from_day:
                            new_cell[key] = row[key]
                            continue

                    if isinstance(value, dict):
                        subdict = {}
                        for subkey in ('name', 'label', 'color',
                                       '_id', 'ref', 'taxonomy'):
                            if subkey in value:
                                subdict[subkey] = value[subkey]
                        if not subdict.get('name'):
                            # create a name so we can rely on it,
                            # e.g. in queries
                            subdict['name'] = subdict['_id']

                        new_cell[key] = subdict
                    elif isinstance(value, self.simpletypes):
                        new_cell[key] = value
                # if the row has no amount set 0.0
                amount = row.get('amount')
                new_cell['amount'] = amount and amount or 0.0<commit_msg>Stringify any IDs (such as ObjectIds) before copying into name field<commit_after>import logging
import time
import sys
import math

from collections import defaultdict
from types import NoneType

from openspending import mongo
from openspending import model
from openspending.lib import util
from openspending.lib.aggregator import _aggregation_query

log = logging.getLogger(__name__)


class Cube(object):
    '''A cube that preaggregates entries.

    Cubes are tied to datasets. It can be used if it is defined
    for the dataset::

      {'name': 'exampledataset',
       'cubes': {'examplecube': {'dimensions': ['dim1', 'dim2', 'dim3']}}}

    A cube contains cells for each unique combination of cube dimension
    and measures the amount (`amount`) and the number of entries
    (`num_entries`) of all entries aggregated in the cell.

    Raises: :exc:`AssertionError` if a cube with the name is not defined for
    the dataset.
    '''

    def __init__(self, dataset, cube_name):
        """
        Arguments:
        ``dataset``
            A dict-like ``dataset`` object
        ``cube_name``
            The name of the cube

        Raises: :exc:`ValueError` if the dimensions specified for the
        cube *cube_name* are not allowed for the *dataset*. See
        :meth:`_cube_dimensions`.
        """
        self.dataset = dataset
        self.name = cube_name
        self.cube_description = self._cube_description()
        self.collection_name = 'cubes.%s.%s' % (dataset['name'], cube_name)
        self.dimensions = self._cube_dimensions()
        self.db = mongo.db
        self.simpletypes = (str, unicode, int, float, NoneType)

    def compute(self):
        """
        Create the cube. This processes all entries of the dataset,
        aggregates cells based on the dimensions of the cube and
        saves them into a mongodb collection in the cubes namespace.
        """
        log.info("Computing cube '%s'...", self.name)
        log.debug(" dataset: '%s'\n  dimensions: '%s'",
                  self.dataset['name'], ', '.join(self.dimensions))
        begin = time.time()

        # query fields: We query for all fields, but handle the date
        # have to query for 'time' if dates are involved.
        # time is a required field for entries, and some datasets
        # add a dimension for time, others don't.
        # If we specify cubes, we do it with 'year' (and maybe 'month')
        query_dimensions = set(self.dimensions)
        used_time_dimensions = query_dimensions.intersection(['year', 'month'])
        additional_dimensions = ['amount']
        if used_time_dimensions:
            query_dimensions = query_dimensions - used_time_dimensions
            additional_dimensions.append('time')
        query_dimensions = query_dimensions.union(additional_dimensions)
        cursor = _aggregation_query(self.dataset, {}, fields=list(query_dimensions))

        # remove a collection if there is one
        if self.is_computed():
            self.db.drop_collection(self.collection_name)
        collection = self.db[self.collection_name]

        for row in cursor:

            cell_id = self._cell_id_for_row(row, query_dimensions)
            cell = collection.find_one({'_id': cell_id})

            if cell is None:
                new_cell = {'_id': cell_id}
                for key in query_dimensions:
                    # handle dates especially, collect year and month
                    if key == 'time':
                        if 'year' in used_time_dimensions:
                            value = int(util.deep_get(row, 'time.from.year'))
                            new_cell['year'] = value
                        if 'month' in used_time_dimensions:
                            value = int(util.deep_get(row, 'time.from.month')[-2:])
                            new_cell['month'] = value
                        continue

                    value = util.deep_get(row, key)
                    if isinstance(value, dict):
                        from_day = util.deep_get(value, 'from.day')
                        if from_day:
                            new_cell[key] = row[key]
                            continue

                    if isinstance(value, dict):
                        subdict = {}
                        for subkey in ('name', 'label', 'color',
                                       '_id', 'ref', 'taxonomy'):
                            if subkey in value:
                                subdict[subkey] = value[subkey]
                        if not subdict.get('name'):
                            # create a name so we can rely on it,
                            # e.g. in queries
                            subdict['name'] = str(subdict['_id'])

                        new_cell[key] = subdict
                    elif isinstance(value, self.simpletypes):
                        new_cell[key] = value
                # if the row has no amount set 0.0
                amount = row.get('amount')
                new_cell['amount'] = amount and amount or 0.0
{'fraction_non_alphanumeric': 0.09154090548054011, 'fraction_numerical': 0.0015885623510722795, 'mean_word_length': 1.761996161228407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5970288', 'n_tokens_mistral': 2687, 'n_tokens_neox': 2550, 'n_words': 827}