<commit_before>from __future__ import absolute_import, division, print_function

import pytest

import numpy as np
import pandas as pd
from datetime import datetime, date

from blaze.compute.core import compute, compute_up
from blaze.expr import symbol, by, exp, summary, Broadcast, join, concat
from blaze import sin
from odo import into
from datashape import discover, to_numpy, dshape


x = np.array([(1, 'Alice', 100),
              (2, 'Bob', -200),
              (3, 'Charlie', 300),
              (4, 'Denis', 400),
              (5, 'Edith', -500)],
             dtype=[('id', 'i8'), ('name', 'S7'), ('amount', 'i8')])

t = symbol('t', discover(x))


def eq(a, b):
    c = a == b
    if isinstance(c, np.ndarray):
        return c.all()
    return c


def test_symbol():
    assert eq(compute(t, x), x)


def test_eq():
    assert eq(compute(t['amount'] == 100, x),
              x['amount'] == 100)


def test_selection():
    assert eq(compute(t[t['amount'] == 100], x), x[x['amount'] == 0])
    assert eq(compute(t[t['amount'] < 0], x), x[x['amount'] < 0])


def test_arithmetic():
    assert eq(compute(t['amount'] + t['id'], x),
              x['amount'] + x['id'])
    assert eq(compute(t['amount'] * t['id'], x),
              x['amount'] * x['id'])
    assert eq(compute(t['amount'] % t['id'], x),
              x['amount'] % x['id'])


def test_UnaryOp():
    assert eq(compute(exp(t['amount']), x),
              np.exp(x['amount']))

    assert eq(compute(abs(-t['amount']), x),
              abs(-x['amount']))


def test_Neg():
    assert eq(compute(-t['amount'], x),
              -x['amount'])


def test_invert_not():
    assert eq(compute(~(t.amount > 0), x),
              ~(x['amount'] > 0))


def test_Reductions():
    assert compute(t['amount'].mean(), x) == x['amount'].mean()
    assert compute(t['amount'].count(), x) == len(x['amount'])
    assert compute(t['amount'].sum(), x) == x['amount'].sum()
    assert compute(t['amount'].min(), x) == x['amount'].min()
    assert compute(t['amount'].max(), x) == x['amount'].max()
    assert compute(t['amount'].nunique(), x) == len(np.unique(x['amount']))
    assert compute(t['amount'].var(), x) == x['amount'].var()
    assert compute(t['amount'].std(), x) == x['amount'].std()
    assert compute(t['amount'].var(unbiased=True), x) == x['amount'].var(ddof=1)
    assert compute(t['amount'].std(unbiased=True), x) == x['amount'].std(ddof=1)
    assert compute((t['amount'] > 150).any(), x) == True
    assert compute((t['amount'] > 250).all(), x) == False
    assert compute(t['amount'][0], x) == x['amount'][0]
    assert compute(t['amount'][-1], x) == x['amount'][-1]


def test_count_string():
    s = symbol('name', 'var * ?string')
    x = np.array(['Alice', np.nan, 'Bob', 'Denis', 'Edith'], dtype='object')
    assert compute(s.count(), x) == 4


def test_reductions_on_recarray():
    assert compute(t.count(), x) == len(x)


def test_count_nan():
    t = symbol('t', '3 * ?real')
    x = np.array([1.0, np.nan, 2.0])
    assert compute(t.count(), x) == 2


def test_Distinct():
    x = np.array([('Alice', 100),
                  ('Alice', -200),
                  ('Bob', 100),
                  ('Bob', 100)],
                dtype=[('name', 'S5'), ('amount', 'i8')])

    t = symbol('t', 'var * {name: string, amount: int64}')

    assert eq(compute(t['name'].distinct(), x),
              np.unique(x['name']))
    assert eq(compute(t.distinct(), x),
              np.unique(x))


def test_sort():
    assert eq(compute(t.sort('amount'), x),
              np.sort(x, order='amount'))
<commit_msg>TST: Adds tests for numpy distinct on<commit_after>from __future__ import absolute_import, division, print_function

import pytest

import numpy as np
import pandas as pd
from datetime import datetime, date

from blaze.compute.core import compute, compute_up
from blaze.expr import symbol, by, exp, summary, Broadcast, join, concat
from blaze import sin
from odo import into
from datashape import discover, to_numpy, dshape


x = np.array([(1, 'Alice', 100),
              (2, 'Bob', -200),
              (3, 'Charlie', 300),
              (4, 'Denis', 400),
              (5, 'Edith', -500)],
             dtype=[('id', 'i8'), ('name', 'S7'), ('amount', 'i8')])

t = symbol('t', discover(x))


def eq(a, b):
    c = a == b
    if isinstance(c, np.ndarray):
        return c.all()
    return c


def test_symbol():
    assert eq(compute(t, x), x)


def test_eq():
    assert eq(compute(t['amount'] == 100, x),
              x['amount'] == 100)


def test_selection():
    assert eq(compute(t[t['amount'] == 100], x), x[x['amount'] == 0])
    assert eq(compute(t[t['amount'] < 0], x), x[x['amount'] < 0])


def test_arithmetic():
    assert eq(compute(t['amount'] + t['id'], x),
              x['amount'] + x['id'])
    assert eq(compute(t['amount'] * t['id'], x),
              x['amount'] * x['id'])
    assert eq(compute(t['amount'] % t['id'], x),
              x['amount'] % x['id'])


def test_UnaryOp():
    assert eq(compute(exp(t['amount']), x),
              np.exp(x['amount']))

    assert eq(compute(abs(-t['amount']), x),
              abs(-x['amount']))


def test_Neg():
    assert eq(compute(-t['amount'], x),
              -x['amount'])


def test_invert_not():
    assert eq(compute(~(t.amount > 0), x),
              ~(x['amount'] > 0))


def test_Reductions():
    assert compute(t['amount'].mean(), x) == x['amount'].mean()
    assert compute(t['amount'].count(), x) == len(x['amount'])
    assert compute(t['amount'].sum(), x) == x['amount'].sum()
    assert compute(t['amount'].min(), x) == x['amount'].min()
    assert compute(t['amount'].max(), x) == x['amount'].max()
    assert compute(t['amount'].nunique(), x) == len(np.unique(x['amount']))
    assert compute(t['amount'].var(), x) == x['amount'].var()
    assert compute(t['amount'].std(), x) == x['amount'].std()
    assert compute(t['amount'].var(unbiased=True), x) == x['amount'].var(ddof=1)
    assert compute(t['amount'].std(unbiased=True), x) == x['amount'].std(ddof=1)
    assert compute((t['amount'] > 150).any(), x) == True
    assert compute((t['amount'] > 250).all(), x) == False
    assert compute(t['amount'][0], x) == x['amount'][0]
    assert compute(t['amount'][-1], x) == x['amount'][-1]


def test_count_string():
    s = symbol('name', 'var * ?string')
    x = np.array(['Alice', np.nan, 'Bob', 'Denis', 'Edith'], dtype='object')
    assert compute(s.count(), x) == 4


def test_reductions_on_recarray():
    assert compute(t.count(), x) == len(x)


def test_count_nan():
    t = symbol('t', '3 * ?real')
    x = np.array([1.0, np.nan, 2.0])
    assert compute(t.count(), x) == 2


def test_distinct():
    x = np.array([('Alice', 100),
                  ('Alice', -200),
                  ('Bob', 100),
                  ('Bob', 100)],
                dtype=[('name', 'S5'), ('amount', 'i8')])

    t = symbol('t', 'var * {name: string, amount: int64}')

    assert eq(compute(t['name'].distinct(), x),
              np.unique(x['name']))
    assert eq(compute(t.distinct(), x),
              np.unique(x))


def test_distinct_on():
    rec = pd.DataFrame(
        [[0, 1],
         [0, 2],
         [1, 1],
         [1, 2]],
        columns=('a', 'b'),
    ).to_records(index=False)

    s = symbol('s', 'var * {a: float64, b: float64}')
    assert (
        compute(s.distinct('a'), rec) ==
        pd.DataFrame(
            [[0, 1],
             [1, 1]],
            columns=('a', 'b'),
        ).to_records(index=False)
    ).all()


def test_distinct_on_str():
    rec = pd.DataFrame(
        [['a', 'a'],
         ['a', 'b'],
         ['b', 'a'],
         ['b', 'b']],
        columns=('a', 'b'),
    ).to_records(index=False).astype([('a', '<U1'), ('b', '<U1')])

    s = symbol('s', 'var * {a: float64, b: float64}')
    assert (
        compute(s.distinct('a'), rec) ==
        pd.DataFrame(
            [['a', 'a'],
             ['b', 'a']],
            columns=('a', 'b'),
        ).to_records(index=False).astype([('a', '<U1'), ('b', '<U1')])
    ).all()


def test_sort():
    assert eq(compute(t.sort('amount'), x),
              np.sort(x, order='amount'))

{'fraction_non_alphanumeric': 0.23106153468987498, 'fraction_numerical': 0.020593282667320422, 'mean_word_length': 2.80727951469902, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1873985', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2799, 'n_words': 668}