<commit_before>import pandas as pd
import numpy as np
from functools import singledispatch
from ..siu import Symbolic

def register_symbolic(f):
    @f.register(Symbolic)
    def _dispatch_symbol(__data, *args, **kwargs):
        return cls(Call("__call__", f, __data.source, *args, **kwargs))

    return f


def cumall(): pass

def cumany(): pass

def cummean(): pass

def desc(): pass

def dense_rank(): pass

def percent_rank(): pass

def min_rank(): pass

def cume_dist(): pass


@register_symbolic
@singledispatch
def row_number(x):
    return pd.Series(np.arange(len(x)))


def ntile(): pass

def cume_dist(): pass

def between(): pass

def coalesce(): pass

def lead(): pass

def lag(): pass

def n(): pass

def n_distinct(): pass

def na_if(): pass

def near(): pass

def nth(): pass

def first(): pass

def last(): pass

__ALL__ = ["row_number"]<commit_msg>add most df vector methods<commit_after>import pandas as pd
import numpy as np
from functools import singledispatch
from ..siu import Symbolic, create_sym_call,Call


def register_symbolic(f):
    # TODO: don't use singledispatch if it has already been done
    f = singledispatch(f)
    @f.register(Symbolic)
    def _dispatch_symbol(__data, *args, **kwargs):
        return create_sym_call(f, __data.source, *args, **kwargs)

    return f

def _expand_bool(x, f):
    return x.expanding().apply(f).astype(bool)

@register_symbolic
def cumall(x):
    return _expand_bool(x, np.all)


@register_symbolic
def cumany(x):
    return _expand_bool(x, np.any)


@register_symbolic
def cummean(x):
    return x.expanding().mean()

@register_symbolic
def desc():
    NotImplementedError("Use minus sign in arrange instead (e.g. -_.somecol)")


@register_symbolic
def dense_rank(x):
    return x.rank(method = "dense")


@register_symbolic
def percent_rank(x):
    NotImplementedError("PRs welcome")


@register_symbolic
def min_rank(x):
    return x.rank(method = "min")


@register_symbolic
def cume_dist(x):
    return x.rank(method = "max") / x.count()


@register_symbolic
def row_number(x):
    if isinstance(x, pd.DataFrame):
        n = x.shape[0]
    else:
        n = len(x)
    return np.arange(n)


@register_symbolic
def ntile(x, n):
    NotImplementedError("ntile not implemented")


@register_symbolic
def between(x, left, right):
    # note: NA -> False, in tidyverse NA -> NA
    return x.between(left, right)
    

@register_symbolic
def coalesce(*args):
    NotImplementedError("coalesce not implemented")


@register_symbolic
def lead(x, n = 1, default = None):
    res = x.shift(-1*n)

    if default is not None:
        res.iloc[-n:] = default

    return res


@register_symbolic
def lag():
    res = x.shift(n)

    if default is not None:
        res.iloc[:n] = default

    return res


@register_symbolic
def n(x):
    if isinstance(x, pd.DataFrame):
        return x.shape[0]

    return len(x)


@register_symbolic
def n_distinct(x):
    return len(x.unique())


@register_symbolic
def na_if(x, y):
    y = [y] if not np.ndim(y) else y

    tmp_x = x.copy(deep = True)
    tmp_x[x.isin(y)] = np.nan

    return tmp_x


@register_symbolic
def near():
    NotImplementedError("near not implemented") 


@register_symbolic
def nth():
    NotImplementedError("nth not implemented") 


@register_symbolic
def first():
    NotImplementedError("first not implemented")


@register_symbolic
def last():
    NotImplementedError("last not implemented")
{'fraction_non_alphanumeric': 0.12099125364431487, 'fraction_numerical': 0.0011661807580174927, 'mean_word_length': 3.8255977496483826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471277', 'n_tokens_mistral': 1223, 'n_tokens_neox': 1206, 'n_words': 335}