<commit_before>import importlib
from collections import OrderedDict
from copy import copy

from six import iteritems
from sqlalchemy import Float
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import func
from sqlalchemy import distinct
from sqlalchemy import case
from sqlalchemy.util import lightweight_named_tuple

from yaml import safe_load

from recipe import BadRecipe, Ingredient, BadIngredient
from recipe import Dimension
from recipe import Metric
from recipe.compat import basestring
from recipe.utils import AttrDict


# Ensure case and distinct don't get reaped. We need it in scope for
# creating Metrics
_distinct = distinct
_case = case


def ingredient_class_for_name(class_name):
    # load the module, will raise ImportError if module cannot be loaded
    m = importlib.import_module('recipe.ingredients')
    # get the class, will raise AttributeError if class cannot be found
    c = getattr(m, class_name, None)
    return c


def parse_condition(cond, table, aggregated=False,
                    default_aggregation='sum'):
    """ Create a format string from a condition """
    if cond is None:
        return None

    else:
        if 'field' not in cond:
            raise BadIngredient('field must be defined in condition')
        field = parse_field(cond['field'],
                            table,
                            aggregated=aggregated,
                            default_aggregation=default_aggregation)
        if 'in' in cond:
            condition_expression = getattr(field, 'in_')(tuple(cond['in']))
        elif 'gt' in cond:
            condition_expression = getattr(field, '__gt__')(cond['gt'])
        elif 'gte' in cond:
            condition_expression = getattr(field, '__gte__')(cond['gte'])
        elif 'lt' in cond:
            condition_expression = getattr(field, '__lt__')(cond['lt'])
        elif 'lte' in cond:
            condition_expression = getattr(field, '__lte__')(cond['lte'])
        elif 'eq' in cond:
            condition_expression = getattr(field, '__eq__')(cond['eq'])
        elif 'ne' in cond:
            condition_expression = getattr(field, '__ne__')(cond['ne'])
        else:
            raise BadIngredient('Bad condition')

        return condition_expression


def tokenize(s):
    """ Tokenize a string by splitting it by + and -

    >>> tokenize('this + that')
    ['this', 'PLUS', 'that']

    >>> tokenize('this+that')
    ['this', 'PLUS', 'that']

    >>> tokenize('this+that-other')
    ['this', 'PLUS', 'that', 'SUB', 'other]
    """

    # Crude tokenization
    s = s.replace('+', ' PLUS ').replace('-', ' MINUS ')
    words = [w for w in s.split(' ') if w]
    return words


def parse_field(fld, table, aggregated=True, default_aggregation='sum'):
    """ Parse a field object from yaml into a sqlalchemy expression """
    # An aggregation is a callable that takes a single field expression
    # None will perform no aggregation
    aggregation_lookup = {
        'sum': func.sum,
        'min': func.min,
        'max': func.max,
        'avg': func.avg,
        'count': func.count,
        'count_distinct': lambda fld: func.count(distinct(fld)),
        'month': lambda fld: func.date_trunc('month', fld),
        'week': lambda fld: func.date_trunc('week', fld),
        'year': lambda fld: func.date_trunc('year', fld),
        'quarter': lambda fld: func.date_trunc('quarter', fld),
        None: lambda fld: fld,
    }

    # Ensure that the dictionary contains:
    # {
    #     'value': str,
    #     'aggregation': str|None,
    #     'condition': dict|None
    # }
    if isinstance(fld, basestring):
        fld = {
            'value': fld,
        }
    if not isinstance(fld, dict):
        raise BadIngredient('fields must be a string or a dict')
    if 'value' not in fld:
        raise BadIngredient('fields must contain a value')
    if not isinstance(fld['value'], basestring):
        raise BadIngredient('field value must be a string')

    # Ensure a condition
    if 'condition' in fld:
        if not isinstance(fld['condition'], dict) and \
                not fld['condition'] is None:
            raise BadIngredient('condition must be null or an object')
    else:
        fld['condition'] = None

    # Ensure an aggregation
    initial_aggregation = default_aggregation if aggregated else None
    if 'aggregation' in fld:
        if not isinstance(fld['aggregation'], basestring) and \
                not fld['aggregation'] is None:
            raise BadIngredient('aggregation must be null or an string')
    else:
        fld['aggregation'] = initial_aggregation

    value = fld.get('value', None)
    if value is None:
        raise BadIngredient('field value is not defined')

    field_parts = []
    for word in tokenize(value):
        if word in ('MINUS', 'PLUS'):
            field_parts.append(word)
        else:
            if hasattr(table, word):
                field_parts.append(getattr(table, word))
            else:
                raise BadIngredient('{} is not a field in {}'.format(
                    word, table.__name__))
    if len(field_parts) is None:
        raise BadIngredient('field is not defined.')
    # Fields should have an odd number of parts
    if len(field_parts) % 2 != 1:
        raise BadIngredient('field does not have the right number of parts')

    field = field_parts[0]
    if len(field_parts) > 1:
        # if we need to add and subtract from the field
        # join the field parts into pairs, for instance if field parts is
        # [MyTable.first, 'MINUS', MyTable.second, 'PLUS', MyTable.third]
        # we will get two pairs here
        # [('MINUS', MyTable.second), ('PLUS', MyTable.third)]
        for operator, other_field in zip(field_parts[1::2], field_parts[2::2]):
            if operator == 'PLUS':
                field = field.__add__(other_field)
            elif operator == 'MINUS':
                field = field.__sub__(other_field)
            else:
                raise BadIngredient('Unknown operator {}'.format(operator))

    # Handle the aggregator
    aggr = fld.get('aggregation', 'sum')
    if aggr is not None:
        aggr = aggr.strip()

    if aggr not in aggregation_lookup:
        raise BadIngredient('unknown aggregation {}'.format(aggr))

    aggregator = aggregation_lookup[aggr]

    condition = parse_condition(fld.get('condition', None),
                                table=table,
                                aggregated=False,
                                default_aggregation=default_aggregation)

    if condition is not None:<commit_msg>[RECIPE-15] Extend field tokenization to support * and /<commit_after>import importlib
from collections import OrderedDict
from copy import copy

from six import iteritems
from sqlalchemy import Float
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import func
from sqlalchemy import distinct
from sqlalchemy import case
from sqlalchemy.util import lightweight_named_tuple

from yaml import safe_load

from recipe import BadRecipe, Ingredient, BadIngredient
from recipe import Dimension
from recipe import Metric
from recipe.compat import basestring
from recipe.utils import AttrDict


# Ensure case and distinct don't get reaped. We need it in scope for
# creating Metrics
_distinct = distinct
_case = case


def ingredient_class_for_name(class_name):
    # load the module, will raise ImportError if module cannot be loaded
    m = importlib.import_module('recipe.ingredients')
    # get the class, will raise AttributeError if class cannot be found
    c = getattr(m, class_name, None)
    return c


def parse_condition(cond, table, aggregated=False,
                    default_aggregation='sum'):
    """ Create a format string from a condition """
    if cond is None:
        return None

    else:
        if 'field' not in cond:
            raise BadIngredient('field must be defined in condition')
        field = parse_field(cond['field'],
                            table,
                            aggregated=aggregated,
                            default_aggregation=default_aggregation)
        if 'in' in cond:
            condition_expression = getattr(field, 'in_')(tuple(cond['in']))
        elif 'gt' in cond:
            condition_expression = getattr(field, '__gt__')(cond['gt'])
        elif 'gte' in cond:
            condition_expression = getattr(field, '__gte__')(cond['gte'])
        elif 'lt' in cond:
            condition_expression = getattr(field, '__lt__')(cond['lt'])
        elif 'lte' in cond:
            condition_expression = getattr(field, '__lte__')(cond['lte'])
        elif 'eq' in cond:
            condition_expression = getattr(field, '__eq__')(cond['eq'])
        elif 'ne' in cond:
            condition_expression = getattr(field, '__ne__')(cond['ne'])
        else:
            raise BadIngredient('Bad condition')

        return condition_expression


def tokenize(s):
    """ Tokenize a string by splitting it by + and -

    >>> tokenize('this + that')
    ['this', 'PLUS', 'that']

    >>> tokenize('this+that')
    ['this', 'PLUS', 'that']

    >>> tokenize('this+that-other')
    ['this', 'PLUS', 'that', 'SUB', 'other]
    """

    # Crude tokenization
    s = s.replace('+', ' PLUS ').replace('-', ' MINUS ') \
        .replace('/', ' DIVIDE ').replace('*', ' MULTIPLY ')
    words = [w for w in s.split(' ') if w]
    return words


def parse_field(fld, table, aggregated=True, default_aggregation='sum'):
    """ Parse a field object from yaml into a sqlalchemy expression """
    # An aggregation is a callable that takes a single field expression
    # None will perform no aggregation
    aggregation_lookup = {
        'sum': func.sum,
        'min': func.min,
        'max': func.max,
        'avg': func.avg,
        'count': func.count,
        'count_distinct': lambda fld: func.count(distinct(fld)),
        'month': lambda fld: func.date_trunc('month', fld),
        'week': lambda fld: func.date_trunc('week', fld),
        'year': lambda fld: func.date_trunc('year', fld),
        'quarter': lambda fld: func.date_trunc('quarter', fld),
        None: lambda fld: fld,
    }

    # Ensure that the dictionary contains:
    # {
    #     'value': str,
    #     'aggregation': str|None,
    #     'condition': dict|None
    # }
    if isinstance(fld, basestring):
        fld = {
            'value': fld,
        }
    if not isinstance(fld, dict):
        raise BadIngredient('fields must be a string or a dict')
    if 'value' not in fld:
        raise BadIngredient('fields must contain a value')
    if not isinstance(fld['value'], basestring):
        raise BadIngredient('field value must be a string')

    # Ensure a condition
    if 'condition' in fld:
        if not isinstance(fld['condition'], dict) and \
                not fld['condition'] is None:
            raise BadIngredient('condition must be null or an object')
    else:
        fld['condition'] = None

    # Ensure an aggregation
    initial_aggregation = default_aggregation if aggregated else None
    if 'aggregation' in fld:
        if not isinstance(fld['aggregation'], basestring) and \
                not fld['aggregation'] is None:
            raise BadIngredient('aggregation must be null or an string')
    else:
        fld['aggregation'] = initial_aggregation

    value = fld.get('value', None)
    if value is None:
        raise BadIngredient('field value is not defined')

    field_parts = []
    for word in tokenize(value):
        if word in ('MINUS', 'PLUS', 'DIVIDE', 'MULTIPLY'):
            field_parts.append(word)
        else:
            if hasattr(table, word):
                field_parts.append(getattr(table, word))
            else:
                raise BadIngredient('{} is not a field in {}'.format(
                    word, table.__name__))
    if len(field_parts) is None:
        raise BadIngredient('field is not defined.')
    # Fields should have an odd number of parts
    if len(field_parts) % 2 != 1:
        raise BadIngredient('field does not have the right number of parts')

    field = field_parts[0]
    if len(field_parts) > 1:
        # if we need to add and subtract from the field
        # join the field parts into pairs, for instance if field parts is
        # [MyTable.first, 'MINUS', MyTable.second, 'PLUS', MyTable.third]
        # we will get two pairs here
        # [('MINUS', MyTable.second), ('PLUS', MyTable.third)]
        for operator, other_field in zip(field_parts[1::2], field_parts[2::2]):
            if operator == 'PLUS':
                field = field.__add__(other_field)
            elif operator == 'MINUS':
                field = field.__sub__(other_field)
            elif operator == 'DIVIDE':
                field = field.__div__(other_field)
            elif operator == 'MULTIPLY':
                field = field.__mul__(other_field)
            else:
                raise BadIngredient('Unknown operator {}'.format(operator))

    # Handle the aggregator
    aggr = fld.get('aggregation', 'sum')
    if aggr is not None:
        aggr = aggr.strip()

    if aggr not in aggregation_lookup:
        raise BadIngredient('unknown aggregation {}'.format(aggr))

    aggregator = aggregation_lookup[aggr]

    condition = parse_condition(fld.get('condition', None),
                                table=table,
                                aggregated=False,
                                default_aggregation=default_aggregation)

    if condition is not None:
{'fraction_non_alphanumeric': 0.10653681570016231, 'fraction_numerical': 0.0013280212483399733, 'mean_word_length': 2.458790507782598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2599317', 'n_tokens_mistral': 3969, 'n_tokens_neox': 3834, 'n_words': 1234}