<commit_before>import logging
import json
import decimal
import copy
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

__all__ = [
    "AggregationBrowser",
    "Cell",
    "Cut",
    "PointCut",
    "RangeCut",
    "SetCut",
    "AggregationResult",
    "cuts_from_string",
    "string_from_cuts",
    "string_from_path",
    "path_from_string"
]

class AggregationBrowser(object):
    """Class for browsing data cube aggregations

    :Attributes:
      * `cube` - cube for browsing

    """

    def __init__(self, cube):
        super(AggregationBrowser, self).__init__()

        if not cube:
            raise AttributeError("No cube given for aggregation browser")

        self.cube = cube

    def full_cube(self):
        return Cell(self.cube)

    def dimension_object(self, dimension):
        """Helper function to return proper dimension object as a subclass of Dimension.

        .. Warning::

            Depreciated. Use :meth:`cubes.Cube.dimension`

        :Arguments:
            * `dimension` - a dimension object or a string, if it is a string, then dimension
              object is retrieved from cube
        """

        if type(dimension) == str:
            return self.cube.dimension(dimension)
        else:
            return dimension

    def aggregate(self, cell, measures = None, drilldown = None, **options):
        """Return aggregate of a cell.

        Subclasses of aggregation browser should implement this method.

        :Attributes:

            * `drilldown` - dimensions and levels through which to drill-down, default `None`
            * `measures` - list of measures to be aggregated. By default all measures are
              aggregated.
            
        Drill down can be specified in two ways: as a list of dimensions or as a dictionary. If it
        is specified as list of dimensions, then cell is going to be drilled down on the next
        level of specified dimension. Say you have a cell for year 2010 and you want to drill
        down by months, then you specify ``drilldown = ["date"]``.
        
        If `drilldown` is a dictionary, then key is dimension or dimension name and value is last
        level to be drilled-down by. If the cell is at `year` level and drill down is: ``{
        "date": "day" }`` then both `month` and `day` levels are added.
        
        If there are no more levels to be drilled down, an exception is raised. Say your model has
        three levels of the `date` dimension: `year`, `month`, `day` and you try to drill down by
        `date` then ``ValueError`` will be raised.
        
        Retruns a :class:AggregationResult object.
        """
        raise NotImplementedError
        
    # def crosstab(self, cell, measures, drilldown, rows = None, columns = None, **options):
    #     """Aggregate facts and return cross-table. Calls `aggregate()` to get results then
    #     transforms the `AggregationResult` into a cross-tab.
    # 
    #     :Parameters:
    #         * cell - cell to be aggregated
    #         * rows - list of dimensions to be put on rows
    #         * columns - list of dimensions to be put on columns
    #     """
    #     raise NotImplementedError("cross tab is under development, body of this function should be considered as development notes")
    #     
    #     drilldown = []
    # 
    #     if rows:
    #         for dim in rows:
    #             dim = self.cube.dimension(dim)

    def facts(self, cell, **options):
        """Return an iterable object with of all facts within cell"""
        
        raise NotImplementedError

    def fact(self, key):
        """Returns a single fact from cube specified by fact key `key`"""
        raise NotImplementedError

    def values(self, cell, dimension, depth = None, paths = None, **options):
        """Return values for `dimension` with level depth `depth`. If `depth` is ``None``, all
        levels are returned.
        
        .. note::
            
            Currently only default hierarchy is used. 
        """
        raise NotImplementedError
        
    def report(self, cell, report):
        """Creates multiple outputs specified in the `report`.
        
        `report` is a dictionary with multiple aggregation browser queries. Keys are custom names
        of queries which requestor can later use to retrieve respective query result. Values are
        dictionaries specifying single query arguments. Each query should contain at least
        one required value ``query`` which contains name of the query function: ``aggregate``,
        ``facts``, ``fact`` or ``values``. Rest of values are function specific, please refer to
        the respective function documentation for more information.
        
        Result is a dictionary where keys wil lbe the query names specified in report
        specification and values will be result values from each query call.

        This method provides convenient way to perform multiple common queries at once, for example
        you might want to have always on a page: total transaction count, total transaction amount,
        drill-down by year and drill-down by transaction type.

        *Roll-up*
        
        Report queries might contain ``rollup`` specification which will result in "rolling-up"
        one or more dimensions to desired level. This functionality is provided for cases when you
        would like to report at higher level of aggregation than the cell you provided is in.
        It works in similar way as drill down in :meth:`AggregationBrowser.aggregate` but in
        the opposite direction (it is like ``cd ..`` in a UNIX shell).
        
        Example: You are reporting for year 2010, but you want to have a bar chart with all years.
        You specify rollup::

            ...
            "rollup": "date",
            ...
        
        Roll-up can be:
        
            * a string - single dimension to be rolled up one level
            * an array - list of dimension names to be rolled-up one level
            * a dictionary where keys are dimension names and values are levels to be rolled up-to
        
        *Future*
        
        In the future there might be optimisations added to this method, therefore it will become
        faster than subsequent separate requests. Also when used with Slicer OLAP service server
        number of HTTP call overhead is reduced.
        """
        
        report_result = {}
        
        for result_name, report_query in report.items():
            query = report_query.get("query")
            if not query:
                raise KeyError("No report query for '%s'" % result_name)
                
            args = dict(report_query)
            del args["query"]
            
            # Note: we do not just convert name into function from symbol for possible future
            # more fine-tuning of queries as strings

            # Handle rollup
            rollup = report_query.get("rollup")
            if rollup:
                query_cell = cell.rollup(rollup)
            else:
                query_cell = cell

            if query == "aggregate":
                result = self.aggregate(query_cell, **args)
            elif query == "facts":
                result = self.facts(query_cell, **args)
            elif query == "fact":
                # Be more tolerant: by default we want "key", but "id" might be common
                key = args.get("key")
                if not key:
                    key = args.get("id")
                result = self.fact(key)
            elif query == "values":
                result = self.values(query_cell, **args)
            elif query == "drilldown":
                raise NotImplementedError("Drill-down queries are not yet implemented")
            else:
                raise KeyError("Unknown report query '%s' for '%s'" % (query, result_name))

            report_result[result_name] = result
            
        return report_result
        
        
class Cell(object):
    """Part of a cube determined by slicing dimensions. Immutable object."""
    def __init__(self, cube=None, cuts=[]):
        self.cube = cube
        self.cuts = cuts

    def slice(self, dimension, path):
        """Create another cell by slicing receiving cell through `dimension` at `path`.
        Receiving object is not modified. If cut with dimension exists it is replaced with new one.
        If path is empty list or is none, then cut for given dimension is removed.

        Example::

            full_cube = Cell(cube)
            contracts_2010 = full_cube.slice("date", [2010])

        Returns: new derived cell object.
        """
        dimension = self.cube.dimension(dimension)
        cuts = self._filter_dimension_cuts(dimension, exclude=True)
        if path:
            cut = PointCut(dimension, path)
            cuts.append(cut)
        return Cell(cube=self.cube, cuts=cuts)

    def drilldown(self, dimension, value):
        """Create another cell by drilling down `dimension` next level on
        current level's key `value`.

        Example::

            cell = cubes.Cell(cube)
            cell = cell.drilldown("date", 2010)
            cell = cell.drilldown("date", 1)
            
        is equivalent to:
        
            cut = cubes.PointCut("date", [2010, 1])
            cell = cubes.Cell(cube, [cut])

        Reverse operation is ``cubes.rollup("date")``
        
        Works only if the cut for dimension is `PointCut`. Otherwise the
        behaviour is undefined.

        Returns: new derived cell object.
        """
        dimension = self.cube.dimension(dimension)
        dim_cut = self.cut_for_dimension(dimension)

        old_path = dim_cut.path if dim_cut else []
        new_cut = PointCut(dimension, old_path + [value])
        
        cuts = [cut for cut in self.cuts if cut is not dim_cut]
        cuts.append(new_cut)

        return Cell(cube=self.cube, cuts=cuts)

    # def cut(self, cuts):
    #     """Cretes another cell by cutting with multiple cuts. `cut` can be a :class:`cubes.Cut`
    #     subclass instance or list of such instances."""
    #     
    # raise NotImplementedError()
            

    def multi_slice(self, cuts):
        """Create another cell by slicing through multiple slices. `cuts` can be list or a dictionry.
        If it is a list, it should be a list of two item tuples where first item is a dimension, second
        item is a dimension cut path. If `cuts` is a dictionary, then keys are dimensions, values are
        cut paths.

        See :meth:`Cell.slice` for more information about slicing."""

        cell = self

        if type(cuts) == dict:
            for dim, path in cuts.items():
                cell = cell.slice(dim, path)
        elif type(cuts) == list or type(cuts) == tuple:
            for dim, path in cuts:
                cell = cell.slice(dim, path)
        else:
            raise TypeError("Cuts for multi_slice sohuld be a list or a dictionary, is '%s'" \
                                % cuts.__class__)

        return cell

    def cut_for_dimension(self, dimension):
        """Return first found cut for given `dimension`"""
        dimension = self.cube.dimension(dimension)

        cut_dimension = None
        for cut in self.cuts:
            cut_dimension = self.cube.dimension(cut.dimension)

            if cut_dimension == dimension:
                return cut

        return None

    def rollup_dim(self, dimension, level=None):
        """Rolls-up cell - goes one or more levels up through dimension
        hierarchy.

        .. note::

                Only default hierarchy is currently supported.
        """

        # FIXME: make this the default roll-up
        # Reason:
        #     * simpler to use
        #     * can be used more nicely in Jinja templates

        dimension = self.cube.dimension(dimension)
        dim_cut = self.cut_for_dimension(dimension)

        if not dim_cut:
            return copy.copy(self)
            # raise ValueError("No cut to roll-up for dimension '%s'" % dimension.name)
        if type(dim_cut) != PointCut:
            raise NotImplementedError("Only PointCuts are currently supported for "
                                      "roll-up (rollup dimension: %s)" % dimension.name)

        cuts = [cut for cut in self.cuts if cut is not dim_cut]

        hier = dimension.default_hierarchy
        rollup_path = hier.rollup(dim_cut.path, level)
        new_cut = PointCut(dimension, rollup_path)
        cuts.append(new_cut)

        return Cell(cube=self.cube, cuts=cuts)

    def rollup(self, rollup):
        """Rolls-up cell - goes one or more levels up through dimension hierarchy. It works in
        similar way as drill down in :meth:`AggregationBrowser.aggregate` but in the opposite
        direction (it is like ``cd ..`` in a UNIX shell).
        
        Roll-up can be:
        
            * a string - single dimension to be rolled up one level
            * an array - list of dimension names to be rolled-up one level
            * a dictionary where keys are dimension names and values are levels to be rolled up-to

        .. note::
            
                Only default hierarchy is currently supported.
        """

        # FIXME: rename this to something like really_complex_rollup :-)
        # Reason:
        #     * see reasons above for rollup_dim()
        #     * used only by Slicer server
<commit_msg>Remove cut from dimension if we are at the top in cell.rollup; add dimension to each constructed attribute in model; added preliminary implementation of attribute.ref()<commit_after>import logging
import json
import decimal
import copy
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

__all__ = [
    "AggregationBrowser",
    "Cell",
    "Cut",
    "PointCut",
    "RangeCut",
    "SetCut",
    "AggregationResult",
    "cuts_from_string",
    "string_from_cuts",
    "string_from_path",
    "path_from_string"
]

class AggregationBrowser(object):
    """Class for browsing data cube aggregations

    :Attributes:
      * `cube` - cube for browsing

    """

    def __init__(self, cube):
        super(AggregationBrowser, self).__init__()

        if not cube:
            raise AttributeError("No cube given for aggregation browser")

        self.cube = cube

    def full_cube(self):
        return Cell(self.cube)

    def dimension_object(self, dimension):
        """Helper function to return proper dimension object as a subclass of Dimension.

        .. Warning::

            Depreciated. Use :meth:`cubes.Cube.dimension`

        :Arguments:
            * `dimension` - a dimension object or a string, if it is a string, then dimension
              object is retrieved from cube
        """

        if type(dimension) == str:
            return self.cube.dimension(dimension)
        else:
            return dimension

    def aggregate(self, cell, measures = None, drilldown = None, **options):
        """Return aggregate of a cell.

        Subclasses of aggregation browser should implement this method.

        :Attributes:

            * `drilldown` - dimensions and levels through which to drill-down, default `None`
            * `measures` - list of measures to be aggregated. By default all measures are
              aggregated.
            
        Drill down can be specified in two ways: as a list of dimensions or as a dictionary. If it
        is specified as list of dimensions, then cell is going to be drilled down on the next
        level of specified dimension. Say you have a cell for year 2010 and you want to drill
        down by months, then you specify ``drilldown = ["date"]``.
        
        If `drilldown` is a dictionary, then key is dimension or dimension name and value is last
        level to be drilled-down by. If the cell is at `year` level and drill down is: ``{
        "date": "day" }`` then both `month` and `day` levels are added.
        
        If there are no more levels to be drilled down, an exception is raised. Say your model has
        three levels of the `date` dimension: `year`, `month`, `day` and you try to drill down by
        `date` then ``ValueError`` will be raised.
        
        Retruns a :class:AggregationResult object.
        """
        raise NotImplementedError
        
    # def crosstab(self, cell, measures, drilldown, rows = None, columns = None, **options):
    #     """Aggregate facts and return cross-table. Calls `aggregate()` to get results then
    #     transforms the `AggregationResult` into a cross-tab.
    # 
    #     :Parameters:
    #         * cell - cell to be aggregated
    #         * rows - list of dimensions to be put on rows
    #         * columns - list of dimensions to be put on columns
    #     """
    #     raise NotImplementedError("cross tab is under development, body of this function should be considered as development notes")
    #     
    #     drilldown = []
    # 
    #     if rows:
    #         for dim in rows:
    #             dim = self.cube.dimension(dim)

    def facts(self, cell, **options):
        """Return an iterable object with of all facts within cell"""
        
        raise NotImplementedError

    def fact(self, key):
        """Returns a single fact from cube specified by fact key `key`"""
        raise NotImplementedError

    def values(self, cell, dimension, depth = None, paths = None, **options):
        """Return values for `dimension` with level depth `depth`. If `depth` is ``None``, all
        levels are returned.
        
        .. note::
            
            Currently only default hierarchy is used. 
        """
        raise NotImplementedError
        
    def report(self, cell, report):
        """Creates multiple outputs specified in the `report`.
        
        `report` is a dictionary with multiple aggregation browser queries. Keys are custom names
        of queries which requestor can later use to retrieve respective query result. Values are
        dictionaries specifying single query arguments. Each query should contain at least
        one required value ``query`` which contains name of the query function: ``aggregate``,
        ``facts``, ``fact`` or ``values``. Rest of values are function specific, please refer to
        the respective function documentation for more information.
        
        Result is a dictionary where keys wil lbe the query names specified in report
        specification and values will be result values from each query call.

        This method provides convenient way to perform multiple common queries at once, for example
        you might want to have always on a page: total transaction count, total transaction amount,
        drill-down by year and drill-down by transaction type.

        *Roll-up*
        
        Report queries might contain ``rollup`` specification which will result in "rolling-up"
        one or more dimensions to desired level. This functionality is provided for cases when you
        would like to report at higher level of aggregation than the cell you provided is in.
        It works in similar way as drill down in :meth:`AggregationBrowser.aggregate` but in
        the opposite direction (it is like ``cd ..`` in a UNIX shell).
        
        Example: You are reporting for year 2010, but you want to have a bar chart with all years.
        You specify rollup::

            ...
            "rollup": "date",
            ...
        
        Roll-up can be:
        
            * a string - single dimension to be rolled up one level
            * an array - list of dimension names to be rolled-up one level
            * a dictionary where keys are dimension names and values are levels to be rolled up-to
        
        *Future*
        
        In the future there might be optimisations added to this method, therefore it will become
        faster than subsequent separate requests. Also when used with Slicer OLAP service server
        number of HTTP call overhead is reduced.
        """
        
        report_result = {}
        
        for result_name, report_query in report.items():
            query = report_query.get("query")
            if not query:
                raise KeyError("No report query for '%s'" % result_name)
                
            args = dict(report_query)
            del args["query"]
            
            # Note: we do not just convert name into function from symbol for possible future
            # more fine-tuning of queries as strings

            # Handle rollup
            rollup = report_query.get("rollup")
            if rollup:
                query_cell = cell.rollup(rollup)
            else:
                query_cell = cell

            if query == "aggregate":
                result = self.aggregate(query_cell, **args)
            elif query == "facts":
                result = self.facts(query_cell, **args)
            elif query == "fact":
                # Be more tolerant: by default we want "key", but "id" might be common
                key = args.get("key")
                if not key:
                    key = args.get("id")
                result = self.fact(key)
            elif query == "values":
                result = self.values(query_cell, **args)
            elif query == "drilldown":
                raise NotImplementedError("Drill-down queries are not yet implemented")
            else:
                raise KeyError("Unknown report query '%s' for '%s'" % (query, result_name))

            report_result[result_name] = result
            
        return report_result
        
        
class Cell(object):
    """Part of a cube determined by slicing dimensions. Immutable object."""
    def __init__(self, cube=None, cuts=[]):
        self.cube = cube
        self.cuts = cuts

    def slice(self, dimension, path):
        """Create another cell by slicing receiving cell through `dimension` at `path`.
        Receiving object is not modified. If cut with dimension exists it is replaced with new one.
        If path is empty list or is none, then cut for given dimension is removed.

        Example::

            full_cube = Cell(cube)
            contracts_2010 = full_cube.slice("date", [2010])

        Returns: new derived cell object.
        """
        dimension = self.cube.dimension(dimension)
        cuts = self._filter_dimension_cuts(dimension, exclude=True)
        if path:
            cut = PointCut(dimension, path)
            cuts.append(cut)
        return Cell(cube=self.cube, cuts=cuts)

    def drilldown(self, dimension, value):
        """Create another cell by drilling down `dimension` next level on
        current level's key `value`.

        Example::

            cell = cubes.Cell(cube)
            cell = cell.drilldown("date", 2010)
            cell = cell.drilldown("date", 1)
            
        is equivalent to:
        
            cut = cubes.PointCut("date", [2010, 1])
            cell = cubes.Cell(cube, [cut])

        Reverse operation is ``cubes.rollup("date")``
        
        Works only if the cut for dimension is `PointCut`. Otherwise the
        behaviour is undefined.

        Returns: new derived cell object.
        """
        dimension = self.cube.dimension(dimension)
        dim_cut = self.cut_for_dimension(dimension)

        old_path = dim_cut.path if dim_cut else []
        new_cut = PointCut(dimension, old_path + [value])
        
        cuts = [cut for cut in self.cuts if cut is not dim_cut]
        cuts.append(new_cut)

        return Cell(cube=self.cube, cuts=cuts)

    # def cut(self, cuts):
    #     """Cretes another cell by cutting with multiple cuts. `cut` can be a :class:`cubes.Cut`
    #     subclass instance or list of such instances."""
    #     
    # raise NotImplementedError()
            

    def multi_slice(self, cuts):
        """Create another cell by slicing through multiple slices. `cuts` can be list or a dictionry.
        If it is a list, it should be a list of two item tuples where first item is a dimension, second
        item is a dimension cut path. If `cuts` is a dictionary, then keys are dimensions, values are
        cut paths.

        See :meth:`Cell.slice` for more information about slicing."""

        cell = self

        if type(cuts) == dict:
            for dim, path in cuts.items():
                cell = cell.slice(dim, path)
        elif type(cuts) == list or type(cuts) == tuple:
            for dim, path in cuts:
                cell = cell.slice(dim, path)
        else:
            raise TypeError("Cuts for multi_slice sohuld be a list or a dictionary, is '%s'" \
                                % cuts.__class__)

        return cell

    def cut_for_dimension(self, dimension):
        """Return first found cut for given `dimension`"""
        dimension = self.cube.dimension(dimension)

        cut_dimension = None
        for cut in self.cuts:
            cut_dimension = self.cube.dimension(cut.dimension)

            if cut_dimension == dimension:
                return cut

        return None

    def rollup_dim(self, dimension, level=None):
        """Rolls-up cell - goes one or more levels up through dimension
        hierarchy. If there is no level to go up (we are at the top level),
         then the cut is removed.
         
         Returns new cell object.

        .. note::

                Only default hierarchy is currently supported.
        """

        # FIXME: make this the default roll-up
        # Reason:
        #     * simpler to use
        #     * can be used more nicely in Jinja templates

        dimension = self.cube.dimension(dimension)
        dim_cut = self.cut_for_dimension(dimension)

        if not dim_cut:
            return copy.copy(self)
            # raise ValueError("No cut to roll-up for dimension '%s'" % dimension.name)
        if type(dim_cut) != PointCut:
            raise NotImplementedError("Only PointCuts are currently supported for "
                                      "roll-up (rollup dimension: %s)" % dimension.name)

        cuts = [cut for cut in self.cuts if cut is not dim_cut]

        hier = dimension.default_hierarchy
        rollup_path = hier.rollup(dim_cut.path, level)

        # If the rollup path is empty, we are at the top level therefore we
        # are removing the cut for the dimension.

        if rollup_path:
            new_cut = PointCut(dimension, rollup_path)
            cuts.append(new_cut)

        return Cell(cube=self.cube, cuts=cuts)

    def rollup(self, rollup):
        """Rolls-up cell - goes one or more levels up through dimension hierarchy. It works in
        similar way as drill down in :meth:`AggregationBrowser.aggregate` but in the opposite
        direction (it is like ``cd ..`` in a UNIX shell).
        
        Roll-up can be:
        
            * a string - single dimension to be rolled up one level
            * an array - list of dimension names to be rolled-up one level
            * a dictionary where keys are dimension names and values are levels to be rolled up-to

        .. note::
            
                Only default hierarchy is currently supported.
        """

        # FIXME: rename this to something like really_complex_rollup :-)
        # Reason:
        #     * see reasons above for rollup_dim()
        #     * used only by Slicer server

{'fraction_non_alphanumeric': 0.07753738928415856, 'fraction_numerical': 0.0018876143458690286, 'mean_word_length': 1.9866652211621856, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4463999', 'n_tokens_mistral': 7281, 'n_tokens_neox': 6847, 'n_words': 2894}