<jupyter_start><jupyter_text>Benchmark Pandas vs Cudf- Using *timeit*  System details  GPU<jupyter_code>!nvidia-smi -q<jupyter_output>==============NVSMI LOG==============

Timestamp                           : Thu Jul  4 06:05:46 2019
Driver Version                      : 418.56
CUDA Version                        : 10.1

Attached GPUs                       : 1
GPU 00000000:0B:00.0
    Product Name                    : Tesla V100-SXM2-32GB
    Product Brand                   : Tesla
    Display Mode                    : Enabled
    Display Active                  : Disabled
    Persistence Mode                : Disabled
    Accounting Mode                 : Disabled
    Accounting Mode Buffer Size     : 4000
    Driver Model
        Current                     : N/A
        Pending                     : N/A
    Serial Number                   : 0321018116769
    GPU UUID                        : GPU-a2891a13-c0c9-5148-3cfd-b103c41a059b
    Minor Number                    : 3
    VBIOS Version                   : 88.00.80.00.04
    MultiGPU Board                  : No
    Board ID                        : 0xb00
    [...]<jupyter_text>CPU<jupyter_code>!less /proc/cpuinfo<jupyter_output>processor       : 0
vendor_id       : GenuineIntel
cpu family      : 6
model           : 79
model name      : Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz
stepping        : 1
microcode       : 0xb00002e
cpu MHz         : 2909.843
cache size      : 51200 KB
physical id     : 0
siblings        : 40
core id         : 0
cpu cores       : 20
apicid          : 0
initial apicid  : 0
fpu             : yes
fpu_exception   : yes
cpuid level     : 20
wp              : yes
[K:[K<jupyter_text>Preperations- Imports- Variables<jupyter_code>import os
import cudf
import pandas as pd
# Source files
raw_parquet_file = 'logs_sample/haproxy_logs_10k.parquet'
raw_json_file = 'logs_sample/haproxy_json_logs_raw_original.txt'

# Benchmark configuration
nlargest = 300<jupyter_output><empty_output><jupyter_text>Target file size validationSet target size (in MB) for the test file<jupyter_code># Source to be used
source_file = raw_json_file 

# Target file for multiplication
target_file = 'logs_sample/1gb.file' 
target_test_size_in_mb = 1000
multiplication_factor = target_test_size_in_mb // (os.path.getsize(source_file) >> 20)
print(f'Multiply target file by {multiplication_factor}')
with open(source_file, 'r') as source:
    with open(target_file, 'w') as target:
        [target.write(line) 
         for line in source.readlines() 
         for m in range(multiplication_factor)]
!ls -lah logs_sample/<jupyter_output><empty_output><jupyter_text>Benchmark Flow- Read file- Compute aggregations- get nlargest()<jupyter_code>benchmark_file = target_file<jupyter_output><empty_output><jupyter_text>cuDF<jupyter_code>%%timeit

# Read file
# gdf = cudf.read_parquet(benchmark_file)
gdf = cudf.read_json(benchmark_file, lines=True)

# Perform aggregation
gdf.groupby(['log_ip']).agg({'feconn':'mean',
                                'beconn':'mean',
                                'time_backend_response':'max',
                                'time_backend_response':'mean',
                                'time_queue':'mean',
                                'time_duration': 'mean',
                                'time_request': 'mean',
                                'time_backend_connect':'mean'
                               })

# Select top N
nlarge = gdf.nlargest(nlargest, 'time_backend_response')<jupyter_output>1.36 s ± 16.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Pandas<jupyter_code>%%timeit

# Read file
# gdf = pd.read_parquet(benchmark_file)
gdf = pd.read_json(benchmark_file, lines=True)

# Perform aggregation
gdf.groupby(['log_ip']).agg({'feconn':'mean',
                                'beconn':'mean',
                                'time_backend_response':'max',
                                'time_backend_response':'mean',
                                'time_queue':'mean',
                                'time_duration': 'mean',
                                'time_request': 'mean',
                                'time_backend_connect':'mean'
                               })

# Select top N
nlarge = gdf.nlargest(nlargest, 'time_backend_response')<jupyter_output>14.2 s ± 97.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Test loading times  cudf<jupyter_code>%%timeit
# gdf = cudf.read_parquet(benchmark_file)
gdf = cudf.read_json(benchmark_file, lines=True)<jupyter_output>1.12 s ± 8.68 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Pandas<jupyter_code>%%timeit
# gdf = pd.read_parquet(benchmark_file)
gdf = pd.read_json(benchmark_file, lines=True)<jupyter_output>14 s ± 1.22 s per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Test aggregation<jupyter_code># gdf = cudf.read_parquet(benchmark_file)
gdf = cudf.read_json(benchmark_file, lines=True)

# pdf = pd.read_parquet(benchmark_file)
pdf = pd.read_json(benchmark_file, lines=True)<jupyter_output><empty_output><jupyter_text>cudf<jupyter_code>%%timeit
gdf.groupby(['log_ip']).agg({'feconn':'mean',
                                'beconn':'mean',
                                'time_backend_response':'max',
                                'time_backend_response':'mean',
                                'time_queue':'mean',
                                'time_duration': 'mean',
                                'time_request': 'mean',
                                'time_backend_connect':'mean'
                               })
nlarge = gdf.nlargest(nlargest, 'time_backend_response')<jupyter_output>211 ms ± 13.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Pandas<jupyter_code>%%timeit
pdf.groupby(['log_ip']).agg({'feconn':'mean',
                                'beconn':'mean',
                                'time_backend_response':'max',
                                'time_backend_response':'mean',
                                'time_queue':'mean',
                                'time_duration': 'mean',
                                'time_request': 'mean',
                                'time_backend_connect':'mean'
                               })
nlarge = pdf.nlargest(nlargest, 'time_backend_response')<jupyter_output>789 ms ± 44.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
{'fraction_non_alphanumeric': 0.10414717804801996, 'fraction_numerical': 0.02977860929217337, 'mean_word_length': 1.9014020805065581, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '333630', 'license': 'Apache-2.0', 'n_tokens_mistral': 2204, 'n_tokens_neox': 1969, 'n_words': 410, 'path': 'demos/gpu-demos/rapids/benchmark-cudf-vs-pd.ipynb', 'repo_name': 'marcelonyc/tutorials', 'size': '6414'}