<jupyter_start><jupyter_text>pipdaA framework for data piping in pythonInspired by [siuba][1], [dfply][2], [plydata][3] and [dplython][4], but with simple yet powerful APIs to mimic the `dplyr` and `tidyr` packages in python Installation```shellpip install -U pipda``` UsageCheckout [plyrda][6] for more detailed usages. VerbsVerbs are functions next to the piping sign (i.e. `>>`) receiving the data directly.<jupyter_code>try:
    import pandas
except ImportError:
    !pip install pandas
import pandas as pd
from pipda import (
    register_verb, 
    register_func, 
    register_operator, 
    evaluate_expr,
    Operator, 
    Symbolic,
    Context
)

f = Symbolic()

df = pd.DataFrame({
    'x': [0, 1, 2, 3],
    'y': ['zero', 'one', 'two', 'three']
})

df

@register_verb(pd.DataFrame)
def head(data, n=5):
    return data.head(n)

df >> head(2)
@register_verb(pd.DataFrame, context=Context.EVAL)
def mutate(data, **kwargs):
    data = data.copy()
    for key, val in kwargs.items():
        data[key] = val
    return data

df >> mutate(z=1)
df >> mutate(z=f.x)
# Verbs that don't compile f.a to data, but just the column name
@register_verb(pd.DataFrame, context=Context.SELECT)
def select(data, *columns):
    return data.loc[:, columns]

# f.x won't be compiled as df.x but just 'x'
df >> mutate(z=2*f.x) >> select(f.x, f.z)

# Compile the args inside the verb
@register_verb(pd.DataFrame, context=Context.PENDING)
def mutate_existing(data, column, value):
    column = evaluate_expr(column, data, Context.SELECT)
    value = evaluate_expr(value, data, Context.EVAL)
    data = data.copy()
    data[column] = value
    return data

# First f.x compiled as column name, and second as Series data
df2 = df >> mutate_existing(f.x, 10 * f.x)
df2
# Evaluate the arguments by yourself
@register_verb(pd.DataFrame, context=Context.PENDING)
def mutate_existing2(data, column, value):
    column = evaluate_expr(column, data, Context.SELECT)
    value = evaluate_expr(value, df2, Context.EVAL)
    data = data.copy()
    data[column] = value
    return data

df >> mutate_existing2(f.x, 2 * f.x)
# register for multiple types
@register_verb(int, context=Context.EVAL)
def add(data, other):
    return data + other

# add is actually a singledispatch generic function
@add.register(float, context=Context.EVAL)
def _(data, other):
    return data * other

1 >> add(1)

1.1 >> add(1.0)

# As it's a singledispatch generic function, we can do it for multiple types
# with the same logic
@register_verb(context=Context.EVAL)
def mul(data, other):
    raise NotImplementedError # not invalid until types registered

@mul.register(int, context=Context.EVAL)
@mul.register(float, context=Context.EVAL) 
# or you could do @mul.register((int, float), context=Context.EVAL)
# context is also supported
def _(data, other):
    return data * other

3 >> mul(2)
3.2 >> mul(2)<jupyter_output><empty_output><jupyter_text>Functions used in verb arguments<jupyter_code>@register_func(context=Context.EVAL)
def if_else(data, cond, true, false):
    cond.loc[cond.isin([True]), ] = true
    cond.loc[cond.isin([False]), ] = false
    return cond

# The function is then also a singledispatch generic function

df >> mutate(z=if_else(f.x>1, 20, 10))
# function without data argument
@register_func(None)
def length(strings):
    return [len(s) for s in strings]

df >> mutate(z=length(f.y))
# register existing functions
from numpy import vectorize
len = register_func(None, context=Context.EVAL, func=vectorize(len))

# original function still works
print(len('abc'))

df >> mutate(z=len(f.y))<jupyter_output>3<jupyter_text>OperatorsYou may also redefine the behavior of the operators<jupyter_code>@register_operator
class MyOperators(Operator):
    def xor(self, a, b):
        """Inteprete X ^ Y as pow(X, Y)."""
        return a ** b

df >> mutate(z=f.x ^ 2)<jupyter_output><empty_output><jupyter_text>ContextThe context defines how a reference (`f.A`, `f['A']`, `f.A.B` is evaluated)<jupyter_code>from pipda import ContextBase

class MyContext(ContextBase):
    name = 'my'
    def getattr(self, parent, ref):
        # double it to distinguish getattr
        return getattr(parent, ref)
    def getitem(self, parent, ref):
        return parent[ref] * 2
    @property
    def ref(self):
        # how we evaluate the ref in f[ref]
        return self 
    
    
@register_verb(context=MyContext())
def mutate_mycontext(data, **kwargs):
    for key, val in kwargs.items():
        data[key] = val
    return data

df >> mutate_mycontext(z=f.x + f['x'])
    
# when ref in f[ref] is also needed to be evaluated
df = df >> mutate(zero=0, one=1, two=2, three=3)
df
df >> mutate_mycontext(m=f[f.y][:1].values[0])
# f.y returns ['zero', 'one', 'two', 'three']
# f[f.y] gets [[0, 2, 4, 6], [0, 2, 4, 6], [0, 2, 4, 6], [0, 2, 4, 6]]
# f[f.y][:1].values gets [[0, 4, 8, 16]]
# f[f.y][:1].values[0] returns [0, 8, 16, 32]
# Notes that each subscription ([]) will double the values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1382488479262673, 'fraction_numerical': 0.015227409336806252, 'mean_word_length': 4.227225130890052, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '9405', 'license': 'MIT', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1742, 'n_words': 518, 'path': 'README.ipynb', 'repo_name': 'pwwang/dpipe', 'size': '4991'}