<jupyter_start><jupyter_text>SkData Steps The OpenRefine json file for steps was used as reference:```json[  {    "op": "core/text-transform",    "description": "Text transform on cells in column Sex using expression grel:if(value == 'male', 1, 0)",    "engineConfig": {      "mode": "row-based",      "facets": []    },    "columnName": "Sex",    "expression": "grel:if(value == 'male', 1, 0)",    "onError": "keep-original",    "repeat": false,    "repeatCount": 10  }]```**SkData Steps format proposed:**```json[  {    'data-set': 'data_set_name', // required    'operation': 'categorize|text-transform|fill-na|drop-na|drop-unique', // required    'column': 'column_name',  // optional    'new-column': 'new_column_name', // optional    'expression': 'dict|inline-if' // required  }]```<jupyter_code>from IPython.display import display
from functools import reduce
# local
from skdata.cleaning import *
from skdata.data import summary

import json
import pandas as pd
def replace(value: str, replace_dict: dict):
    """
    """
    if not isinstance(value, str):
        return value
    
    return reduce(
        lambda x, y: x.replace(y, replace_dict[y]), replace_dict, value
    )
def expr(dataset: pd.DataFrame, step: str):
    # aliases
    op = step['operation']
    k = step['column'] if 'column' in step else None
    k_new = k if 'new-column' not in step else step['new-column']
    c_expr = step['expression']
    data = dataset
    
    if op == 'text-transform':
        f_expr = eval('lambda value: %s' % c_expr)
        dataset[k_new] = dataset[k].apply(f_expr)
        
    elif op == 'categorize':
        params = dict(data=dataset, col_name=k, categories=eval(c_expr))
        params.update(
            {'new_col_name': k_new} if 'new-column' in step else {}
        )
        categorize(**params)
        
    elif op == 'fill-na':
        fill = c_expr
        if c_expr in ['mean', 'max', 'min', 'median']:
            fill = dataset.eval('%s.%s()' % (k, c_expr))
        dataset[k].fillna(fill, inplace=True)
        
    elif op == 'drop-na':
        params = eval(c_expr)
        dropna(dataset, **params)
        
    elif op == 'drop-unique':
        params = eval(c_expr)
        drop_columns_with_unique_values(dataset, **params)
        
    return dataset
steps_json = '''[{
    "data-set": "data",
    "operation": "text-transform",
    "column": "Sex",
    "expression": "value.title()"
}, {
    "data-set": "data",
    "operation": "categorize",
    "column": "Pclass",
    "new-column": "Pclass-name",
    "expression": "{1: 'High Class', 2: 'Middle Class', 3: 'Low Class'}"
}, {
    "data-set": "data",
    "operation": "text-transform",
    "column": "Embarked",
    "expression": "replace(value, {'C': 'Cherbourg', 'Q': 'Queenstown', 'S': 'Southampton'})"
}, {
    "data-set": "data",
    "operation": "fill-na",
    "column": "Age",
    "expression": "mean"
}, {
    "data-set": "data",
    "operation": "drop-na",
    "expression": "{'axis': 1, 'max_na_values': 0.1}"
}, {
    "data-set": "data",
    "operation": "drop-na",
    "expression": "{'axis': 0}"
}, {
    "data-set": "data",
    "operation": "drop-unique",
    "expression": "{'max_unique_values': 0.25}"
}]
'''
def compute(steps, start: int=None, end: int=None, steps_id: list=None):
    """
    
    """
    
    if steps_id is not None:
        _steps = [s for i, s in enumerate(steps) if i in steps_id]
    else:
        _steps = steps[start:end]
    
    for step in _steps:
        globals()[step['data-set']] = expr(globals()[step['data-set']], step)
data = pd.read_csv('../data/train.csv')
steps = json.loads(steps_json)

display(summary(data))

compute(steps)
    
display(data.head())
display(summary(data))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18407032498668088, 'fraction_numerical': 0.004262120404901438, 'mean_word_length': 3.028969957081545, 'pattern_counts': {'":': 36, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '96190', 'license': 'MIT', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1268, 'n_words': 309, 'path': 'notebooks/Steps.ipynb', 'repo_name': 'xmnlab/vizpydata', 'size': '3754'}