<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import datetime as dt
import matplotlib.pylab as plt
dataset_url='https://raw.githubusercontent.com/martin-fabbri/colab-notebooks/master/data/wowah_data.csv'
df = pd.read_csv(dataset_url)
df.head()
def loggg(f):
  def wrapper(df, *args, **kwargs):
    tic = dt.datetime.now()
    result = f(df, *args, **kwargs)
    toc = dt.datetime.now()
    n_char = result.groupby('char').agg(n_char=('char', np.size)).shape
    print(f'{f.__name__} took: {toc - tic} shape={result.shape} n_char={n_char[0]}')
    return result
  return wrapper

@loggg
def start_pipeline(df):
  return df.copy()

@loggg
def clean_dataset(df):
  df.columns = [col_name.lstrip() for col_name in df.columns ]
  return df

@loggg
def remove_outliers(df):
  return df

clean_df = (df
 .pipe(start_pipeline)
 .pipe(clean_dataset)
 .pipe(remove_outliers))
df.columns

df.columns = [col_name.lstrip() for col_name in df.columns ]
df.columns
df2 = (df
 .groupby('char')
 .agg(n_row=('level', np.size),
      min_level=('level', np.min),
      max_level=('level', np.max),
      guild=('guild', np.max)))
df2.head()
df3 = (df.groupby('guild').agg(n_char=('char', 'nunique')))
df3.head()
df4 = df2.merge(df3.reset_index())
df.head()
plt.scatter(x=df4['n_row'], y=df4['level_inc']);
df4['level_inc'] = df4['max_level'] - df4['min_level']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17826704545454544, 'fraction_numerical': 0.009232954545454546, 'mean_word_length': 4.920168067226891, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '534748', 'license': 'Apache-2.0', 'n_tokens_mistral': 568, 'n_tokens_neox': 564, 'n_words': 105, 'path': 'jupyter_best_practices_part_1.ipynb', 'repo_name': 'martin-fabbri/colab-notebooks', 'size': '1408'}