<jupyter_start><jupyter_text>simple pandas vs pandas: DataFrame<jupyter_code>from spandas import DataFrame
import pandas as pd
import time
import numpy as np
import math<jupyter_output><empty_output><jupyter_text>Append<jupyter_code>df = pd.DataFrame({'a': [], 'b': []})
df_append = pd.DataFrame({'a': [1], 'b': [1]})
start = time.time()
for i in range(1000):
    df = df.append(df_append)
end = time.time()
time_df = end - start
fdf = DataFrame({'a': [], 'b': []})
fdf_append = DataFrame({'a': [1], 'b': [1]})
start = time.time()
for i in range(1000):
    fdf = fdf.append(fdf_append)
end = time.time()
time_fdf = end - start
assert(len(fdf) == len(df))
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))
start = time.time()
df = df.append([df_append for _ in range(10000)])
end = time.time()
time_df = end - start
start = time.time()
fdf = fdf.append([fdf_append for _ in range(10000)])
end = time.time()
time_fdf = end - start
assert(len(fdf) == len(df))
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.6971, simple pandas time: 0.028922
boost times: 24.103<jupyter_text>iterrows<jupyter_code>d = {'a': np.zeros(10000), 'b': np.zeros(10000), 'c': np.zeros(10000)}
df = pd.DataFrame(d)
fdf = DataFrame(d)
start = time.time()
for k, v in df.iterrows():
    pass
end = time.time()
time_df = end - start
start = time.time()
for k, v in fdf.iterrows():
    pass
end = time.time()
time_fdf = end - start
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.42995, simple pandas time: 0.011965
boost times: 35.935<jupyter_text>apply<jupyter_code>d = {'math': np.random.randint(50, 100, size=100000), 
     'chinese': np.random.randint(50, 100, size=100000), 
     'english': np.random.randint(50, 100, size=100000)}
df = pd.DataFrame(d)
fdf = DataFrame(d)<jupyter_output><empty_output><jupyter_text>element-wise<jupyter_code>start = time.time()
tmp = df.applymap(lambda x: math.sqrt(x)*10)
end = time.time()
time_df = end - start
start = time.time()
tmp = fdf.apply(lambda x: math.sqrt(x)*10)
end = time.time()
time_df = end - start
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.10568, simple pandas time: 0.011965
boost times: 8.8329<jupyter_text>during the test, something insteresting happens:<jupyter_code>start = time.time()
tmp = df.applymap(lambda x: np.sqrt(x)*10)
end = time.time()
time_df = end - start
start = time.time()
tmp = fdf.apply(lambda x: np.sqrt(x)*10)
end = time.time()
time_df = end - start
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.50967, simple pandas time: 0.011965
boost times: 42.597<jupyter_text>for numpy function, pandas will take more time.  row-wise<jupyter_code>start = time.time()
df['sum'] = df.apply(lambda x: x['math'] +x['chinese'] + x['english'], axis=1)
end = time.time()
time_df = end - start
start = time.time()
fdf[['sum']] = fdf.apply(lambda x: x['math'] +x['chinese'] + x['english'], type='row')
end = time.time()
time_fdf = end - start
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 2.8633, simple pandas time: 0.15658
boost times: 18.286<jupyter_text>column-wise<jupyter_code>start = time.time()
for i in range(100):
    tmp = df.apply(np.mean, axis=0)
end = time.time()
time_df = end - start
time_df
start = time.time()
for i in range(100):
    tmp = fdf.apply(np.mean, type='column')
end = time.time()
time_fdf = end - start
time_fdf
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.39697, simple pandas time: 0.050899
boost times: 7.7991<jupyter_text>Sample<jupyter_code>d = {'math': np.random.randint(50, 100, size=1000000), 
     'chinese': np.random.randint(50, 100, size=1000000), 
     'english': np.random.randint(50, 100, size=1000000)}
df = pd.DataFrame(d)
fdf = DataFrame(d)
start = time.time()
tmp = df.sample(10000)
end = time.time()
time_df = end - start
time_df
start = time.time()
tmp = fdf.sample(10000)
end = time.time()
time_fdf = end - start
time_fdf
len(tmp)
print("pandas time: {:.5}, simple pandas time: {:.5}".format(time_df, time_fdf))
print("boost times: {:.5}".format(time_df / time_fdf))<jupyter_output>pandas time: 0.03894, simple pandas time: 0.010972
boost times: 3.549
{'fraction_non_alphanumeric': 0.18412895226286422, 'fraction_numerical': 0.057449886340152925, 'mean_word_length': 5.293888166449935, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '430316', 'license': 'MIT', 'n_tokens_mistral': 2155, 'n_tokens_neox': 1928, 'n_words': 448, 'path': 'dataframe_time.ipynb', 'repo_name': 'zhuzilin/simple-pandas', 'size': '4839'}