<commit_before># -*- coding: utf-8 -*-

"""Classes that facilitate transformation and aggregation of dataframes.

This module provides wrappers to simplify grouping and aggregation 
of dataframes by index, and to consequently perform arithmetics on 
the aggregated dataframes.

Examples:

    The following code would take a Pandas dataframe ``df``, and produce
    a dataframe ``res_df`` that is indexed by ``name``. The result would have
    a column ``count`` that gives the number of rows per ``name``; a column
    that gives the sum of ``score`` per ``name``, divided by the sum of 
    ``arrests`` per ``name``; and finally two columns that give the sum 
    of ``score`` per ``name``, and the sum of squared ``score`` per ``name``::

        aggregates = [Count(),
                      Proportion('score','arrests'),
                      Aggregate(['score', lambda x: x.score**2],'sum')
                      ]
        res_df = Aggregator(df, aggregates).aggregate('name')

    Note how we specify all definitions before performing the actual
    aggregation, and how ``Aggregator.aggregate()`` then takes an
    ``index`` to group by.

Aggregator also caches individual transformations of columns, as to 
reduce redundant calculations.

Classes that endusers interface with are Aggregate, Fraction, 
Count, and Proportion (all of which specify outcome columns and row-wise 
aggregation functions), and Aggregator (which takes an input dataframe and an
index by which rows are being grouped).

"""

from itertools import product, chain
from collections import defaultdict

import pandas as pd
import numpy as np

from . import util, data
from .data import Column
      
class ColumnReduction(object):
    """Wraps and hashes a `Column` together with a function that aggregates across rows.
    """

    def __init__(self, column, agg_func):
        """Args:
            column (Column): The column that will be aggregated across rows.
            agg_func (Function): The function that will be used to aggregate rows,
                                 once the Column has been grouped by some index.
        """
        if not isinstance(column, Column):
            raise ValueError("ColumnReduction needs a Column")
        self.column = column
        self.agg_func = agg_func

        import pdb; pdb.set_trace()
        # use float32 by default for string agg functions except 'nunique', e.g. 'min', 'median', 'max', etc.
        if isinstance(agg_func, basestring) and agg_func != 'nunique' and self.column.astype is None:
            self.column.astype = np.float32
    def __hash__(self):
        return hash((self.column, self.agg_func))
    def __eq__(self, other):
        return hash(self) == hash(other)
        
class ColumnFunction(object):
    """Abstract base class for functions on reduced Columns; names the outcomes.

    Having obtained Columns that have been created and aggregated along rows, with<commit_msg>remove pdb<commit_after># -*- coding: utf-8 -*-

"""Classes that facilitate transformation and aggregation of dataframes.

This module provides wrappers to simplify grouping and aggregation 
of dataframes by index, and to consequently perform arithmetics on 
the aggregated dataframes.

Examples:

    The following code would take a Pandas dataframe ``df``, and produce
    a dataframe ``res_df`` that is indexed by ``name``. The result would have
    a column ``count`` that gives the number of rows per ``name``; a column
    that gives the sum of ``score`` per ``name``, divided by the sum of 
    ``arrests`` per ``name``; and finally two columns that give the sum 
    of ``score`` per ``name``, and the sum of squared ``score`` per ``name``::

        aggregates = [Count(),
                      Proportion('score','arrests'),
                      Aggregate(['score', lambda x: x.score**2],'sum')
                      ]
        res_df = Aggregator(df, aggregates).aggregate('name')

    Note how we specify all definitions before performing the actual
    aggregation, and how ``Aggregator.aggregate()`` then takes an
    ``index`` to group by.

Aggregator also caches individual transformations of columns, as to 
reduce redundant calculations.

Classes that endusers interface with are Aggregate, Fraction, 
Count, and Proportion (all of which specify outcome columns and row-wise 
aggregation functions), and Aggregator (which takes an input dataframe and an
index by which rows are being grouped).

"""

from itertools import product, chain
from collections import defaultdict

import pandas as pd
import numpy as np

from . import util, data
from .data import Column
      
class ColumnReduction(object):
    """Wraps and hashes a `Column` together with a function that aggregates across rows.
    """

    def __init__(self, column, agg_func):
        """Args:
            column (Column): The column that will be aggregated across rows.
            agg_func (Function): The function that will be used to aggregate rows,
                                 once the Column has been grouped by some index.
        """
        if not isinstance(column, Column):
            raise ValueError("ColumnReduction needs a Column")
        self.column = column
        self.agg_func = agg_func

        # use float32 by default for string agg functions except 'nunique', e.g. 'min', 'median', 'max', etc.
        if isinstance(agg_func, basestring) and agg_func != 'nunique' and self.column.astype is None:
            self.column.astype = np.float32
    def __hash__(self):
        return hash((self.column, self.agg_func))
    def __eq__(self, other):
        return hash(self) == hash(other)
        
class ColumnFunction(object):
    """Abstract base class for functions on reduced Columns; names the outcomes.

    Having obtained Columns that have been created and aggregated along rows, with
{'fraction_non_alphanumeric': 0.0869416395687147, 'fraction_numerical': 0.0020537395173712133, 'mean_word_length': 3.0137362637362637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7614998', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1467, 'n_words': 677}