<jupyter_start><jupyter_text>Imports and data<jupyter_code>from swifter import swiftapply
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>These data (~71 million rows) were taken from https://www.kaggle.com/benhamner/sf-bay-area-bike-share/data<jupyter_code>data = pd.read_feather('/home/ec2-user/data/status')
data['date'] = pd.to_datetime(data['time'])
print(data.shape)
data.head()<jupyter_output>(71984434, 5)<jupyter_text>Necessary functions for timing and plotting<jupyter_code>import dask.dataframe as dd
from dask.multiprocessing import get

def wrapper(func, *args, **kwargs):
    def wrapped():
        return func(*args, **kwargs)
    return wrapped

def pd_apply(df, myfunc, *args, **kwargs):
    def wrapped():
        return df.apply(myfunc, args=args, **kwargs)
    return wrapped

def dask_apply(df, npartitions, myfunc, *args, **kwargs):
    if type(df) == pd.DataFrame:
        kwargs.pop('meta')
        tmp = df.iloc[:1,:].apply(myfunc, args=args, **kwargs)
        meta = {c: tmp[c].dtype for c in tmp.columns}
        return dd.from_pandas(df, npartitions=npartitions).apply(myfunc, *args, axis=1, **kwargs, meta=meta).compute(get=get)
    else:
        meta = kwargs.pop('meta')
        try:
            return dd.from_pandas(df, npartitions=npartitions).map_partitions(myfunc, *args, **kwargs, meta=meta).compute(get=get)
        except:
            return dd.from_pandas(df, npartitions=npartitions).map(lambda x: myfunc(x), *args, **kwargs, meta=meta).compute(get=get)
from psutil import cpu_count
import timeit


def time_all(df, my_func, nrows, *args, **kwargs):
    pd_times = list()
    dask_times = list()
    vectorized_times = list()
    swiftapply_times = list()
    meta = df.iloc[:2].apply(lambda x: my_func(x, *args, **kwargs))
    n_reps = 1

    for rows in nrows:
        print(rows)
        my_sample = df.iloc[:rows]

        print('pd')
        wrapped = pd_apply(my_sample, my_func, *args, **kwargs)
        pd_times.append(timeit.timeit(wrapped, number=n_reps)/n_reps)
                
        print('dask')
        wrapped = wrapper(dask_apply, my_sample, cpu_count()*2, my_func, *args, meta=meta, **kwargs)
        dask_times.append(timeit.timeit(wrapped, number=n_reps)/n_reps)

        try:
            print('vec')
            wrapped = wrapper(my_func, my_sample, *args, **kwargs)
            vectorized_times.append(timeit.timeit(wrapped, number=n_reps)/n_reps)
        except:
            vectorized_times.append(None)

        print('swift')
        wrapped = wrapper(swiftapply, my_sample, my_func, *args, **kwargs)
        swiftapply_times.append(timeit.timeit(wrapped, number=n_reps)/n_reps)
        
    return pd.DataFrame({'pd_apply': pd_times, 'dask_apply': dask_times, 
                         'vectorized': vectorized_times, 'swiftapply': swiftapply_times})

def plot_times(df, log10=False):
    if any([time is not None for time in df['vectorized']]):
        vectorized = True
    else:
        vectorized = False
    if log10:
        plt.plot(np.log10(nrows), np.log10(df['pd_apply']), '-r')
        plt.plot(np.log10(nrows), np.log10(df['dask_apply']), '-b')
        if vectorized:
            plt.plot(np.log10(nrows), np.log10(df['vectorized']), '-c')
        plt.plot(np.log10(nrows), np.log10(df['swiftapply']), '-g')
        plt.xlabel('log10(Rows)')
        plt.ylabel('log10(Time (seconds))')
    else:
        plt.plot(nrows, df['pd_apply'], '-r')
        plt.plot(nrows, df['dask_apply'], '-b')
        if vectorized:
            plt.plot(nrows, df['vectorized'], '-c')
        plt.plot(nrows, df['swiftapply'], '-g')
        plt.xlabel('Rows')
        plt.ylabel('Time (seconds)')
    if vectorized:
        plt.legend(['Pandas apply', 'Dask apply', 'Vectorized', 'swiftapply'])
    else:
        plt.legend(['Pandas apply', 'Dask apply', 'swiftapply'])
    plt.box(on=False)<jupyter_output><empty_output><jupyter_text>Vectorizable functions<jupyter_code>def bikes_proportion(x, max_x):
    return x * 1.0 / max_x
n = data.shape[0]
nrows = [round (n/47828125), round(n/9565625), round(n/1953125), round(n/390625),
              round(n/78125), round(n/15625), round(n/3125),
              round(n/625), round(n/125), round(n/25), round(n/5), n]
%time bikesproportion_times = time_all(data['bikes_available'], bikes_proportion, nrows, max_x=np.max(data['bikes_available']))
#bikes_proportion.to_csv('bikesproportion_times.csv', index=False)
#bikesproportion_times = pd.read_csv('bikesproportion_times.csv')
plot_times(bikesproportion_times, True)
plot_times(bikesproportion_times, False)
n = data.shape[0]
nrows = [round (n/47828125), round(n/9565625), round(n/1953125), round(n/390625), 
              round(n/78125), round(n/15625), round(n/3125),
              round(n/625), round(n/125), round(n/25), round(n/5), n]
%time datetime_times = time_all(data['time'], pd.to_datetime, nrows)
#datetime_times.to_csv('datetime_conversion_times.csv', index=False)
#datetime_times = pd.read_csv('datetime_conversion_times.csv')
plot_times(datetime_times, True)
plot_times(datetime_times, False)<jupyter_output><empty_output><jupyter_text>Non-vectorized functions<jupyter_code>def is_morning(datetime):
    if (datetime.hour >= 6) and (datetime.hour < 12):
        return True
    else:
        return False
n = data.shape[0]
nrows = [round (n/47828125), round(n/9565625), round(n/1953125), round(n/390625),
              round(n/78125), round(n/15625), round(n/3125),
              round(n/625), round(n/125), round(n/25), round(n/5), n]
%time ismorning_times = time_all(data['date'], is_morning, nrows)
#ismorning_times.to_csv('ismorning_times.csv', index=False)
#ismorning_times = pd.read_csv('ismorning_times.csv')
plot_times(ismorning_times, True)
plot_times(ismorning_times, False)
def convert_to_human(datetime):
    return datetime.weekday_name + ', the ' + str(datetime.daysinmonth) + 'th day of ' + datetime.strftime("%B") + ', ' + str(datetime.year)
n = data.shape[0]
nrows = [round (n/47828125), round(n/9565625), round(n/1953125), round(n/390625), 
              round(n/78125), round(n/15625), round(n/3125),
              round(n/625), round(n/125), round(n/25), round(n/5), n]
%time humanreadable_times = time_all(data['date'], convert_to_human, nrows)
#humanreadable_times.to_csv('humanreadable_conversion_times.csv', index=False)
plot_times(humanreadable_times, True)
plot_times(humanreadable_times, False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1544015977876786, 'fraction_numerical': 0.03902289138116454, 'mean_word_length': 3.9430523917995446, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '619762', 'license': 'MIT', 'n_tokens_mistral': 2517, 'n_tokens_neox': 2345, 'n_words': 415, 'path': 'examples/swiftapply_speedcomparison.ipynb', 'repo_name': 'TriggeredMessaging/swifter', 'size': '6509'}