<commit_before>""" MongoDB Backend - Uses aggregation pipeline

If you don't have a mongo server running

    $ conda install mongodb -y
    $ mongod &

>> from blaze import *

>> data = [(1, 'Alice', 100),
...        (2, 'Bob', -200),
...        (3, 'Charlie', 300),
...        (4, 'Denis', 400),
...        (5, 'Edith', -500)]

Migrate data into MongoDB

>> import pymongo
>> db = pymongo.MongoClient().db
>> db.mydata.drop()  # clear out old results
>> _ = into(db.mydata, data, columns=['id', 'name', 'amount'])

Objective: find the name of accounts with negative amount

Using MongoDB query language

>> db.mydata.aggregate([{'$match': {'amount': {'$lt': 0}}}, # doctest: +SKIP
..                      {'$project': {'name': 1, '_id': 0}}])['result']
[{'name': 'Bob'}, {'name': 'Edith'}]

Using Blaze

>> t = Table(db.mydata)
>> t[t.amount < 0].name
    name
0    Bob
1  Edith

>> db.mydata.drop()

Uses the aggregation pipeline
http://docs.mongodb.org/manual/core/aggregation-pipeline/
"""

from __future__ import absolute_import, division, print_function

import operator as op
import numbers

try:
    import pymongo
    from pymongo.collection import Collection
except ImportError:
    Collection = type(None)

import fnmatch
from datashape import Record
from toolz import pluck
import toolz

from ..expr import (var, Label, std, Sort, count, nunique, Selection, mean,
                    Reduction, Head, ReLabel, Apply, Distinct, RowWise, By,
                    TableSymbol, Projection, sum, min, max, TableExpr,
                    Gt, Lt, Ge, Le, Eq, Ne, ScalarSymbol, And, Or, Summary,
                    Like, Relational, Arithmetic, ColumnWise)
from ..expr.core import Expr

from ..dispatch import dispatch


__all__ = ['MongoQuery']


class MongoQuery(object):
    """
    A Pair of a pymongo collection and a aggregate query

    We need to carry around both a pymongo collection and a list of
    dictionaries to feed into the aggregation pipeline.  This class
    carries around such a pair.

    Parameters
    ----------
    coll: pymongo.collection.Collection
        A single pymongo collection, holds a table
    query: list of dicts
        A query to send to coll.aggregate

    >>> q = MongoQuery(db.my_collection, # doctest: +SKIP
    ...     [{'$match': {'name': 'Alice'}},
    ...      {'$project': {'name': 1, 'amount': 1, '_id': 0}}])
    """
    def __init__(self, coll, query):
        self.coll = coll
        self.query = tuple(query)

    def append(self, clause):
        return MongoQuery(self.coll, self.query + (clause,))

    def info(self):
        return self.coll, self.query

    def __eq__(self, other):
        return type(self) == type(other) and self.info() == other.info()

    def __hash__(self):
        return hash((type(self), self.info()))


@dispatch((var, Label, std, Sort, count, nunique, Selection, mean, Reduction,
           Head, ReLabel, Apply, Distinct, RowWise, By, Like), Collection)
def compute_up(e, coll, **kwargs):
    return compute_up(e, MongoQuery(coll, []))


@dispatch(TableSymbol, Collection)
def compute_up(t, coll, **kwargs):
    return MongoQuery(coll, [])


@dispatch(Head, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$limit': t.n})


@dispatch(ColumnWise, MongoQuery)
def compute_one(t, q, **kwargs):
    return q.append(compute_one(t.expr, q, **kwargs))


@dispatch((basestring, ScalarSymbol))
def m(s):
    return '$%s' % s


@dispatch(numbers.Number)
def m(s):
    return s


binops = {op.add: 'add',
          op.mul: 'multiply',
          op.div: 'divide',
          op.sub: 'subtract'}


def op_label(t):
    return '%s(%s, %s)' % (t.op.__name__, t.lhs, t.rhs)


@dispatch(Arithmetic, MongoQuery)
def compute_one(t, q, **kwargs):
    lhs, rhs = t.lhs, t.rhs
    mexpr = {m('project'): {str(t): {m(binops[t.op]): [m(lhs), m(rhs)]}}}
    return mexpr


@dispatch(Projection, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$project': dict((col, 1) for col in t.columns)})


@dispatch(Selection, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$match': match(t.predicate.expr)})


@dispatch(Like, MongoQuery)
def compute_up(t, q, **kwargs):
    pats = dict((name, {'$regex': fnmatch.translate(pattern)})<commit_msg>Rename<commit_after>""" MongoDB Backend - Uses aggregation pipeline

If you don't have a mongo server running

    $ conda install mongodb -y
    $ mongod &

>> from blaze import *

>> data = [(1, 'Alice', 100),
...        (2, 'Bob', -200),
...        (3, 'Charlie', 300),
...        (4, 'Denis', 400),
...        (5, 'Edith', -500)]

Migrate data into MongoDB

>> import pymongo
>> db = pymongo.MongoClient().db
>> db.mydata.drop()  # clear out old results
>> _ = into(db.mydata, data, columns=['id', 'name', 'amount'])

Objective: find the name of accounts with negative amount

Using MongoDB query language

>> db.mydata.aggregate([{'$match': {'amount': {'$lt': 0}}}, # doctest: +SKIP
..                      {'$project': {'name': 1, '_id': 0}}])['result']
[{'name': 'Bob'}, {'name': 'Edith'}]

Using Blaze

>> t = Table(db.mydata)
>> t[t.amount < 0].name
    name
0    Bob
1  Edith

>> db.mydata.drop()

Uses the aggregation pipeline
http://docs.mongodb.org/manual/core/aggregation-pipeline/
"""

from __future__ import absolute_import, division, print_function

import operator as op
import numbers

try:
    import pymongo
    from pymongo.collection import Collection
except ImportError:
    Collection = type(None)

import fnmatch
from datashape import Record
from toolz import pluck
import toolz

from ..expr import (var, Label, std, Sort, count, nunique, Selection, mean,
                    Reduction, Head, ReLabel, Apply, Distinct, RowWise, By,
                    TableSymbol, Projection, sum, min, max, TableExpr,
                    Gt, Lt, Ge, Le, Eq, Ne, ScalarSymbol, And, Or, Summary,
                    Like, Arithmetic, ColumnWise)
from ..expr.core import Expr

from ..dispatch import dispatch


__all__ = ['MongoQuery']


class MongoQuery(object):
    """
    A Pair of a pymongo collection and a aggregate query

    We need to carry around both a pymongo collection and a list of
    dictionaries to feed into the aggregation pipeline.  This class
    carries around such a pair.

    Parameters
    ----------
    coll: pymongo.collection.Collection
        A single pymongo collection, holds a table
    query: list of dicts
        A query to send to coll.aggregate

    >>> q = MongoQuery(db.my_collection, # doctest: +SKIP
    ...     [{'$match': {'name': 'Alice'}},
    ...      {'$project': {'name': 1, 'amount': 1, '_id': 0}}])
    """
    def __init__(self, coll, query):
        self.coll = coll
        self.query = tuple(query)

    def append(self, clause):
        return MongoQuery(self.coll, self.query + (clause,))

    def info(self):
        return self.coll, self.query

    def __eq__(self, other):
        return type(self) == type(other) and self.info() == other.info()

    def __hash__(self):
        return hash((type(self), self.info()))


@dispatch((var, Label, std, Sort, count, nunique, Selection, mean, Reduction,
           Head, ReLabel, Apply, Distinct, RowWise, By, Like), Collection)
def compute_up(e, coll, **kwargs):
    return compute_up(e, MongoQuery(coll, []))


@dispatch(TableSymbol, Collection)
def compute_up(t, coll, **kwargs):
    return MongoQuery(coll, [])


@dispatch(Head, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$limit': t.n})


@dispatch(ColumnWise, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append(compute_up(t.expr, q, **kwargs))


@dispatch((basestring, ScalarSymbol))
def m(s):
    return '$%s' % s


@dispatch(numbers.Number)
def m(s):
    return s


binops = {op.add: 'add',
          op.mul: 'multiply',
          op.div: 'divide',
          op.sub: 'subtract'}


def op_label(t):
    return '%s(%s, %s)' % (t.op.__name__, t.lhs, t.rhs)


@dispatch(Arithmetic, MongoQuery)
def compute_up(t, q, **kwargs):
    lhs, rhs = t.lhs, t.rhs
    mexpr = {m('project'): {str(t): {m(binops[t.op]): [m(lhs), m(rhs)]}}}
    return mexpr


@dispatch(Projection, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$project': dict((col, 1) for col in t.columns)})


@dispatch(Selection, MongoQuery)
def compute_up(t, q, **kwargs):
    return q.append({'$match': match(t.predicate.expr)})


@dispatch(Like, MongoQuery)
def compute_up(t, q, **kwargs):
    pats = dict((name, {'$regex': fnmatch.translate(pattern)})
{'fraction_non_alphanumeric': 0.15874323370204754, 'fraction_numerical': 0.007060484819957637, 'mean_word_length': 3.14990234375, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3587519', 'n_tokens_mistral': 2965, 'n_tokens_neox': 2792, 'n_words': 842}