<commit_before>from farmfs.util import empty2dot, compose, concat, concatMap, fmap, identity, irange, invert, count, take, uniq, groupby, curry, uncurry, identify, pipeline, zipFrom
import functools
from collections import Iterator
from farmfs.util import ingest, egest, safetype, rawtype
import pytest

try:
    from itertools import ifilter
except ImportError:
    ifilter=filter

def add(x,y): return x+y
assert add(1,2) == 3

def inc(x): return x+1
assert inc(1) == 2

def even(x): return x%2==0
assert even(2) == True
assert even(1) == False

even_list = functools.partial(ifilter, even)
assert list(even_list([1,2,3,4])) == [2, 4]

def test_empty2dot():
  assert empty2dot([]) == ["."]
  l = [1,2,3]
  assert empty2dot(l) == l

def test_compose():
  inc_add = compose(inc, add)
  assert inc_add(1,2) == 4

def test_concat():
  assert list(concat([[1,2,3],[4,5,6]])) == [1,2,3,4,5,6]
  assert list(concat([[1],[2,3,4]])) == [1,2,3,4]
  assert list(concat([[1,2,3],[4]])) == [1,2,3,4]
  assert list(concat([[1],[1,2],[1,2,3]])) == [1,1,2,1,2,3]

def test_concatMap():
  assert list(concatMap (lambda x:x*[x])([0,1,2,3,3])) == [1, 2, 2, 3, 3, 3, 3, 3, 3]

def test_fmap():
  inc_iter = fmap(inc)
  assert list(inc_iter([1,2,3,4])) == [2, 3, 4, 5]

def test_identity():
  assert identity(5) == 5

def test_groupby():
  #TODO group by may not order results consistenly for asserts.
  assert groupby(even, [1,2,3,4,5,6]) == [(False,[1,3,5]), (True,[2,4,6])]

def test_take():
  assert list(take(3)([1,2,3,4,5])) == [1,2,3]
  assert list(take(3)([1,2])) == [1,2]

def test_uniq():
  assert list(uniq([1,2,3,4])) == [1,2,3,4]
  assert list(uniq([1,2,2,4])) == [1,2,4]
  assert list(uniq([1,2,3,2])) == [1,2,3]

def test_irange():
  assert list(take(3)(irange(0,1))) == [0,1,2]
  assert list(take(3)(irange(0,-1))) == [0,-1,-2]

def test_invert():
  assert invert(1) == False
  assert invert(True) == False
  assert invert(0) == True
  assert invert([]) == True
  assert invert([1]) == False

def test_count():
  assert count(iter([])) == 0
  assert count(iter([1,2,3])) == 3

def test_curries():
  unadd = uncurry(add)
  assert unadd([1,2]) == 3
  readd = curry(unadd)
  assert readd(1,2) == 3

def test_identify():
  called = False
  def foo(x):
    called = True
    return x+1
  id_foo = identify(foo)
  assert id_foo(1) == 1
  assert called

def test_pipeline():
  identity_pipeline = pipeline()
  assert isinstance(identity_pipeline([1,2,3]), Iterator), "identity_pipeline should be an iterator"<commit_msg>add python3 mock test of identify<commit_after>from farmfs.util import empty2dot, compose, concat, concatMap, fmap, identity, irange, invert, count, take, uniq, groupby, curry, uncurry, identify, pipeline, zipFrom
import functools
from collections import Iterator
from farmfs.util import ingest, egest, safetype, rawtype
import pytest
from unittest.mock import Mock

try:
    from itertools import ifilter
except ImportError:
    ifilter=filter

def add(x,y): return x+y
assert add(1,2) == 3

def inc(x): return x+1
assert inc(1) == 2

def even(x): return x%2==0
assert even(2) == True
assert even(1) == False

even_list = functools.partial(ifilter, even)
assert list(even_list([1,2,3,4])) == [2, 4]

def test_empty2dot():
  assert empty2dot([]) == ["."]
  l = [1,2,3]
  assert empty2dot(l) == l

def test_compose():
  inc_add = compose(inc, add)
  assert inc_add(1,2) == 4

def test_concat():
  assert list(concat([[1,2,3],[4,5,6]])) == [1,2,3,4,5,6]
  assert list(concat([[1],[2,3,4]])) == [1,2,3,4]
  assert list(concat([[1,2,3],[4]])) == [1,2,3,4]
  assert list(concat([[1],[1,2],[1,2,3]])) == [1,1,2,1,2,3]

def test_concatMap():
  assert list(concatMap (lambda x:x*[x])([0,1,2,3,3])) == [1, 2, 2, 3, 3, 3, 3, 3, 3]

def test_fmap():
  inc_iter = fmap(inc)
  assert list(inc_iter([1,2,3,4])) == [2, 3, 4, 5]

def test_identity():
  assert identity(5) == 5

def test_groupby():
  #TODO group by may not order results consistenly for asserts.
  assert groupby(even, [1,2,3,4,5,6]) == [(False,[1,3,5]), (True,[2,4,6])]

def test_take():
  assert list(take(3)([1,2,3,4,5])) == [1,2,3]
  assert list(take(3)([1,2])) == [1,2]

def test_uniq():
  assert list(uniq([1,2,3,4])) == [1,2,3,4]
  assert list(uniq([1,2,2,4])) == [1,2,4]
  assert list(uniq([1,2,3,2])) == [1,2,3]

def test_irange():
  assert list(take(3)(irange(0,1))) == [0,1,2]
  assert list(take(3)(irange(0,-1))) == [0,-1,-2]

def test_invert():
  assert invert(1) == False
  assert invert(True) == False
  assert invert(0) == True
  assert invert([]) == True
  assert invert([1]) == False

def test_count():
  assert count(iter([])) == 0
  assert count(iter([1,2,3])) == 3

def test_curries():
  unadd = uncurry(add)
  assert unadd([1,2]) == 3
  readd = curry(unadd)
  assert readd(1,2) == 3

def test_identify():
  mock = Mock(return_value=1)
  foo = identify(mock)
  result = foo(5)
  assert result == 5
  mock.assert_called_once_with(5)

def test_pipeline():
  identity_pipeline = pipeline()
  assert isinstance(identity_pipeline([1,2,3]), Iterator), "identity_pipeline should be an iterator"
{'fraction_non_alphanumeric': 0.2069850039463299, 'fraction_numerical': 0.06748224151539069, 'mean_word_length': 5.159173754556501, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '189106', 'n_tokens_mistral': 2187, 'n_tokens_neox': 2120, 'n_words': 493}