<jupyter_start><jupyter_text>Testing the loading times<jupyter_code>%%timeit
import pandas as pd
pd.read_csv("Reviews.csv")
%%timeit
import csv
with open('Reviews.csv', encoding="utf-8") as csvfile:
        reader = csv.reader(csvfile)
        data = [row for row in reader]<jupyter_output>5.27 s ± 72 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Experiment 1: Looking for chocolateGoal of this experiment is to get the reviewscores for all reviews containg the word Chocolate (or any other words) in a list.  Python standard<jupyter_code>with open('Reviews.csv', encoding="utf-8") as csvfile:
        import csv
        reader = csv.reader(csvfile)
        data = [row for row in reader]
def python_standard_choc_1(data):
    import csv

    def parse_row_chocolate(row, word):
        if word in row[9].lower():
            return row[6]

    total = []
        
    for row in data:
        result = parse_row_chocolate(row, "chocolate")
        if result:
            total.append((result))
    return len(total)
%%timeit
python_standard_choc_1(data)
def python_standard_choc_2(data):
    import csv

    def parse_row_chocolate(row, word):
        if word in row[9].lower():
            return row[6]

    return len([x for x in map(lambda row: parse_row_chocolate(row, "chocolate"), data) if x])
%%timeit
python_standard_choc_2(data)
def python_standard_choc_3(data):
    import csv
    return len([row[6] for row in data if "chocolate" in row[9].lower()])
%%timeit
python_standard_choc_3(data)<jupyter_output>487 ms ± 2.51 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Pandas<jupyter_code>import pandas as pd
df_load = pd.read_csv("Reviews.csv")
def pandas_choc_1(df):
    return len(list(df[df.Text.str.contains("chocolate", case=False, regex=False, na=False)].Score))
%%timeit
pandas_choc_1(df_load)
def pandas_choc_2(df):
    total = []
    for index, row in df.iterrows():
        if "chocolate" in row.Text.lower():
            total.append(row["Score"])
    return len(total)
%%timeit
pandas_choc_2(df_load)
def pandas_choc_3(df):    
    total = []
    for row in df.itertuples():
        if "chocolate" in row.Text.lower():
            total.append(row.Score)
    return len(total)
%%timeit
pandas_choc_3(df_load)
def pandas_choc_4(df):    
    def parse_chocolate_col(col):
        if "chocolate" in col.lower():
            return True
        return False
    return len(list(df[df["Text"].apply(lambda col: parse_chocolate_col(col))]['Score']))
%%timeit
pandas_choc_4(df_load)
def pandas_choc_5(df):   
    def parse_chocolate_col(col):
        if "chocolate" in col.lower():
            return True
        return False
    return len(list(df[df["Text"].map(lambda col: parse_chocolate_col(col))]['Score']))
%%timeit
pandas_choc_5(df_load)
def pandas_choc_6(df):
    df["succes"] = df.Text.str.lower().str.contains("chocolate")
    return df.succes
%%timeit
pandas_choc_6(df_load)<jupyter_output>1.16 s ± 19.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Numpy<jupyter_code>def numpy_choc_1(df):
    import numpy as np
    def testfunc(arr):
        if "chocolate" in arr.lower():
            return True
        return False

    tfunc = np.vectorize(testfunc)
    return len(list(df[tfunc(df.Text)].Score))
%%timeit
numpy_choc_1(df_load)
def numpy_choc_2(df):
    import numpy as np
    def testfunc(arr):
        if "chocolate" in arr.lower():
            return True
        return False

    tfunc = np.vectorize(testfunc)
    return len(list(df[tfunc(df.Text.values)].Score))
%%timeit
numpy_choc_2(df_load)<jupyter_output>516 ms ± 2.29 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Numba<jupyter_code>from numba import jit
import numpy as np

x = np.arange(100).reshape(10, 10)

@jit(nopython=True) # Set "nopython" mode for best performance, equivalent to @njit
def go_fast(a): # Function is compiled to machine code when called the first time
    trace = 0
    for i in range(a.shape[0]):   # Numba likes loops
        trace += np.tanh(a[i, i]) # Numba likes NumPy functions
    return a + trace              # Numba likes NumPy broadcasting

print(go_fast(x))
from numba import jit
# import csv

@jit
def testfunc(arr):
        if "chocolate" in arr.lower():
            return True
        return False
def numba_choc_1(df=None, df_bool=False):
    import numpy as np
    
    tfunc = np.vectorize(testfunc)
    return len(list(df[tfunc(df.Text.values)].Score))

%%timeit
numba_choc_1(df_load)<jupyter_output>5.19 s ± 151 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Cython<jupyter_code>%load_ext Cython
%%cython --annotate
cdef int a = 0
for i in range(10):
    a += i
print(a)
%load_ext Cython

with open('Reviews.csv', encoding="utf-8") as csvfile:
        import csv
        reader = csv.reader(csvfile)
        data = [row for row in reader]
%%cython
cpdef str parse_row_chocolate(list row, str word):
        if word in row[9].lower():
            return row[6]

cpdef int cython_standard_choc_1(list data):        
    cdef list total = []
        
    for row in data:
        result = parse_row_chocolate(row, "chocolate")
        if result:
            total.append((result))
    return len(total)
%%timeit
cython_standard_choc_1(data)
%%cython
cpdef int cython_standard_choc_3(list data):
    return len([row[6] for row in data if "chocolate" in row[9].lower()])
%%timeit
cython_standard_choc_3(data)
%%cython
# Bint stands for boolean int
cpdef bint testfunc(str arr):
        if "chocolate" in arr.lower():
            return True
        return False

cpdef int cython_numpy_choc_1(df):
    import numpy as np
    tfunc = np.vectorize(testfunc)
    return len(list(df[tfunc(df.Text.values)].Score))
%%timeit
cython_numpy_choc_1(df_load)<jupyter_output>480 ms ± 11 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Experiment 2: Liking what?Goal of the experiment is to calculate the std deviation of the ratings.- Convert string to int- Calculate the sum  Regular Python<jupyter_code>with open('Reviews.csv', encoding="utf-8") as csvfile:
    import csv
    reader = csv.reader(csvfile)
    next(reader)
    data = [row for row in reader]
def python_standard_score_1(reader):
    import math

    total = []

    for row in reader:
        total.append(int(row[6]))

    average = sum(total) / len(total)
    total2 = []
    for value in total:
        total2.append((value - average) ** 2)

    std_dev = math.sqrt((1 / (len(total2) -1) * sum(total2)))

    return std_dev
%%timeit
python_standard_score_1(data)
def python_standard_score_2(reader):
    import math

    def calc_difference(value, average):
        return (value - average) ** 2
        
    scores = [int(row[6]) for row in reader]
    average = sum(scores) / len(scores)

    deviations = list(map(lambda score: calc_difference(score, average), scores))       
    return math.sqrt((1 / (len(deviations) - 1) * sum(deviations)))
%%timeit
python_standard_score_2(data)
def python_standard_score_2B(reader):
    import math

    def calc_difference(value, average):
        return (value - average) ** 2
    
    scores = [int(row[6]) for row in reader]
    average = sum(scores) / len(scores)

    deviations = map(lambda score: calc_difference(score, average), scores)      
    return math.sqrt((1 / (len(scores) - 1) * sum(deviations)))
%%timeit
python_standard_score_2B(data)
## Interesting find
def python_standard_score_3(reader):
    import statistics
        
    scores = [int(row[6]) for row in reader]
    return statistics.stdev(scores)
%%timeit
python_standard_score_3(data)
def python_standard_score_4(reader):
    import math
    
    scores = [int(row[6]) for row in reader]
    average = sum(scores) / len(scores)

    deviations = [(value - average) ** 2 for value in scores]      
    return math.sqrt((1 / (len(deviations) - 1) * sum(deviations)))
%%timeit
python_standard_score_4(data)
data[0]<jupyter_output><empty_output><jupyter_text>Pandas<jupyter_code>import pandas as pd
df = pd.read_csv("Reviews.csv")
def pandas_score_1(df): 
    import math
    
    deviations = (df.Score - df.Score.mean()) ** 2     
    return math.sqrt((1 / (len(deviations) - 1) * sum(deviations)))
%%timeit
pandas_score_1(df)
def pandas_score_2(df): 
    import math
    
    deviations = (df.Score - df.Score.mean()) ** 2     
    return math.sqrt((1 / (len(deviations) - 1) * deviations.sum()))
%%timeit
pandas_score_2(df)
def pandas_score_3(df):
    return df.Score.std()
%%timeit
pandas_score_3(df)<jupyter_output>7.33 ms ± 157 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)<jupyter_text>Numpy<jupyter_code>def numpy_score_1(df):
    return df.Score.values.std()
%%timeit
numpy_score_1(df)
def numpy_score_2(df): 
    import math
    
    deviations = (df.Score.values - df.Score.values.mean()) ** 2     
    return math.sqrt((1 / (len(deviations) - 1) * deviations.sum()))
%%timeit
numpy_score_2(df)<jupyter_output>5.36 ms ± 35.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)<jupyter_text>Numba<jupyter_code>with open('Reviews.csv', encoding="utf-8") as csvfile:
    import csv
    reader = csv.reader(csvfile)
    next(reader)
    data = [row for row in reader]
from numba import jit
import math

@jit
def numba_score_1(reader):
    
    scores = [int(row[6]) for row in reader]
    average = sum(scores) / len(scores)

    deviations = [(value - average) ** 2 for value in scores]      
    return math.sqrt((1 / (len(deviations) - 1) * sum(deviations)))
%%timeit
numba_score_1(data)
import math

@jit
def numba_score_2(scores):
    average = sum(scores) / len(scores)

    deviations = [(value - average) ** 2 for value in scores]      
    return math.sqrt((1 / (len(deviations) - 1) * sum(deviations)))
%%timeit
numba_score_2(df.Score.values.astype(int))<jupyter_output><ipython-input-4-357b6f70b1c5>:3: NumbaWarning: [1m
Compilation is falling back to object mode WITH looplifting enabled because Function "numba_score_2" failed type inference due to: [1mUntyped global name 'sum':[0m [1m[1mcannot determine Numba type of <class 'builtin_function_or_method'>[0m
[1m
File "<ipython-input-4-357b6f70b1c5>", line 5:[0m
[1mdef numba_score_2(scores):
[1m    average = sum(scores) / len(scores)
[0m    [1m^[0m[0m
[0m[0m
  @jit
c:\users\svw-b\envs\big_data\lib\site-packages\numba\object_mode_passes.py:178: NumbaWarning: [1mFunction "numba_score_2" was compiled in object mode without forceobj=True.
[1m
File "<ipython-input-4-357b6f70b1c5>", line 4:[0m
[1m@jit
[1mdef numba_score_2(scores):
[0m[1m^[0m[0m
[0m
  state.func_ir.loc))
c:\users\svw-b\envs\big_data\lib\site-packages\numba\object_mode_passes.py:188: NumbaDeprecationWarning: [1m
Fall-back from the nopython compilation path to the object mode compilation path has been detected, this is[...]<jupyter_text>Cython<jupyter_code>%load_ext Cython
%%cython
cpdef float cython_standard_score_1(list reader):
    import math

    cdef list total = []

    for row in reader:
        total.append(int(row[6]))

    cdef float average = sum(total) / len(total)
    cdef list total2 = []
    for value in total:
        total2.append((value - average) ** 2)

    cdef float std_dev = math.sqrt((1 / (len(total2) -1) * sum(total2)))

    return std_dev
%%timeit
cython_standard_score_1(data)
%%cython
cpdef float cython_pandas_score_2(df): 
    import math
    
    deviations = (df.Score - df.Score.mean()) ** 2     
    return math.sqrt((1 / (len(deviations) - 1) * deviations.sum()))
%%timeit
cython_pandas_score_2(df)
%%cython
cpdef float cython_numpy_score_2(df): 
    import math
    
    deviations = (df.Score.values - df.Score.values.mean()) ** 2     
    return math.sqrt((1 / (len(deviations) - 1) * deviations.sum()))
%%timeit
cython_numpy_score_2(df)
%%cython
cpdef float cython_numpy_score_1(df):
    return df.Score.values.std()
%%timeit
cython_numpy_score_1(df)<jupyter_output>3.54 ms ± 188 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)<jupyter_text>Experiment 3: Sorting it out (text)Sort the summaries of the review on alphabetically ascending  Regular Python<jupyter_code>with open('Reviews.csv', encoding="utf-8") as csvfile:
    import csv
    reader = csv.reader(csvfile)
    next(reader)
    data = [row for row in reader]
def python_standard_sorting_1(reader):
    summaries = []
    for row in reader:
        summaries.append(row[8])
    return sorted(summaries)
%%timeit
python_standard_sorting_1(data)
def python_standard_sorting_2(reader):
    return sorted([row[8] for row in reader])
%%timeit
python_standard_sorting_2(data)
# def qsort(inlist):
#     if inlist == []: 
#         return []
#     else:
#         pivot = inlist[0]
#         lesser = qsort([x for x in inlist[1:] if x < pivot])
#         greater = qsort([x for x in inlist[1:] if x >= pivot])
#         return lesser + [pivot] + greater
# %%time
# qsort([row[8] for row in data])<jupyter_output><empty_output><jupyter_text>Pandas<jupyter_code>import pandas as pd
df = pd.read_csv("Reviews.csv")
def pandas_sorting_1(df):
    return df.Summary.sort_values()
%%timeit
pandas_sorting_1(df)
def pandas_sorting_2(df):
    return sorted(list(df.Summary.astype(str)))
%%timeit
pandas_sorting_2(df)
def pandas_sorting_3(df):
    return sorted(df.Summary.astype(str))
%%timeit
pandas_sorting_3(df)<jupyter_output>386 ms ± 8.58 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Numpy<jupyter_code>def numpy_sorting_1(df):
    import numpy as np
    return np.sort(df.Summary.astype(str).values)
%%timeit
numpy_sorting_1(df)
def numpy_sorting_2(df):
    return sorted(df.Summary.astype(str).values)
%%timeit
numpy_sorting_2(df)<jupyter_output>353 ms ± 10.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Numba<jupyter_code>from numba import jit
# From https://github.com/lprakash/Sorting-Algorithms/blob/master/sorts.ipynb
@jit
def quicksort(a, s, e):
    #print(a, s, e)
    #print(a.__repr__)
    if (e-s)==0:
        return 
    pivot = a[e-1]
    p1 = s
    p2 = e - 1
    while (p1 != p2):
        if (a[p1] > pivot):
            a[p2] = a[p1]
            a[p1] = a[p2-1]
            a[p2-1] = pivot
            p2 = p2 -1
        else: 
            p1+=1
    quicksort(a, s, p2)
    quicksort(a, p2+1, e)
def quicksortwrapper(a):
    quicksort(a, 0, len(a))
%%timeit
a = list([row[8] for row in data])
quicksort(a, 0, 4)<jupyter_output>c:\users\svw-b\envs\big_data\lib\site-packages\numba\ir_utils.py:2041: NumbaPendingDeprecationWarning: [1m
Encountered the use of a type that is scheduled for deprecation: type 'reflected list' found for argument 'a' of function 'quicksort'.

For more information visit http://numba.pydata.org/numba-doc/latest/reference/deprecation.html#deprecation-of-reflection-for-list-and-set-types
[1m
File "<ipython-input-3-e903ddbe35be>", line 3:[0m
[1m@jit
[1mdef quicksort(a, s, e):
[0m[1m^[0m[0m
[0m
  warnings.warn(NumbaPendingDeprecationWarning(msg, loc=loc))<jupyter_text>Cython<jupyter_code>%load_ext Cython
%%cython
cpdef list python_standard_sorting_2(list reader):
    return sorted([row[8] for row in reader])
%%timeit
python_standard_sorting_2(data)<jupyter_output>390 ms ± 2.01 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Experiment 3B: Sorting it out (int)Sort the scores of the reviews  Regular Python<jupyter_code>with open('Reviews.csv', encoding="utf-8") as csvfile:
    import csv
    reader = csv.reader(csvfile)
    next(reader)
    data = [row for row in reader]
def python_standard_sorting_1(reader):
    summaries = []
    for row in reader:
        summaries.append(row[6])
    return sorted(summaries)
%%timeit
python_standard_sorting_1(data)
def python_standard_sorting_1b(reader):
    summaries = []
    for row in reader:
        summaries.append(int(row[6]))
    return sorted(summaries)
%%time
python_standard_sorting_1b(data)
def python_standard_sorting_2(reader):
    return sorted([row[6] for row in reader])
%%timeit
python_standard_sorting_2(data)
# def qsort(inlist):
#     if inlist == []: 
#         return []
#     else:
#         pivot = inlist[0]
#         lesser = qsort([x for x in inlist[1:] if x < pivot])
#         greater = qsort([x for x in inlist[1:] if x >= pivot])
#         return lesser + [pivot] + greater
# %%time
# qsort([int(row[6]) for row in data])<jupyter_output><empty_output><jupyter_text>Pandas<jupyter_code>import pandas as pd
df = pd.read_csv("Reviews.csv")
def pandas_sorting_1(df):
    return df.Score.sort_values()
%%timeit
pandas_sorting_1(df)
def pandas_sorting_2(df):
    return sorted(list(df.Score.astype(int)))
%%timeit
pandas_sorting_2(df)
def pandas_sorting_3(df):
    return sorted(df.Score.astype(int))
%%timeit
pandas_sorting_3(df)<jupyter_output>77.6 ms ± 1.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)<jupyter_text>Numpy<jupyter_code>def numpy_sorting_1(df):
    import numpy as np
    return np.sort(df.Score.values)
%%timeit
numpy_sorting_1(df)
def numpy_sorting_2(df):
    return sorted(df.Score.astype(str).values)
%%timeit
numpy_sorting_2(df)<jupyter_output>391 ms ± 6.63 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Numba<jupyter_code>from numba import jit
# From https://github.com/lprakash/Sorting-Algorithms/blob/master/sorts.ipynb
@jit
def quicksort(a, s, e):
    #print(a, s, e)
    #print(a.__repr__)
    if (e-s)==0:
        return 
    pivot = a[e-1]
    p1 = s
    p2 = e - 1
    while (p1 != p2):
        if (a[p1] > pivot):
            a[p2] = a[p1]
            a[p1] = a[p2-1]
            a[p2-1] = pivot
            p2 = p2 -1
        else: 
            p1+=1
    quicksort(a, s, p2)
    quicksort(a, p2+1, e)
@jit
def sort(l):
    return sorted(l)
%%timeit
a = list([int(row[6]) for row in data])
sort(a)
def quicksortwrapper(a):
    quicksort(a, 0, len(a))
%%timeit
a = list([row[6] for row in data])
quicksort(a, 0, 4)<jupyter_output>c:\users\svw-b\envs\big_data\lib\site-packages\numba\ir_utils.py:2041: NumbaPendingDeprecationWarning: [1m
Encountered the use of a type that is scheduled for deprecation: type 'reflected list' found for argument 'a' of function 'quicksort'.

For more information visit http://numba.pydata.org/numba-doc/latest/reference/deprecation.html#deprecation-of-reflection-for-list-and-set-types
[1m
File "<ipython-input-3-e903ddbe35be>", line 3:[0m
[1m@jit
[1mdef quicksort(a, s, e):
[0m[1m^[0m[0m
[0m
  warnings.warn(NumbaPendingDeprecationWarning(msg, loc=loc))<jupyter_text>Cython<jupyter_code>%load_ext Cython
%%cython
cpdef list cython_standard_sorting_1(list reader):
    return sorted([row[6] for row in reader])
%%timeit
cython_standard_sorting_1(data)<jupyter_output>78.9 ms ± 557 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)<jupyter_text>Extra / Testing area<jupyter_code>import random
listt = [random.randint(0, 100) for x in range(100000)]
%%time
new_list = []
for x in listt:
    new_list.append(x + 1)
new_list[0]
%%time
[x + 1 for x in listt][0]
def addition(n):
    return n + 1
%%time
list(map(lambda x: x + 1, listt))[0]
%%timeit
import csv
with open('Reviews.csv', encoding="utf-8") as csvfile:
        reader = csv.reader(csvfile)
        for x in reader:
            pass
%%timeit
import pandas as pd
pd.read_csv("Reviews.csv")
import csv
test = open('Reviews.csv', encoding="utf8")    
reader = csv.reader(test)
def extra_test(reader):
    def parse_row_chocolate(row, word):
        if word in row[9].lower():
            return row[6]

    total = []
    for row in reader:
        result = parse_row_chocolate(row, "chocolate")
        if result:
            total.append((result))
    return total
%%time
extra_test(reader)
test.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12987474009838226, 'fraction_numerical': 0.02566053045286272, 'mean_word_length': 3.713193116634799, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 81, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '452496', 'license': 'MIT', 'n_tokens_mistral': 7675, 'n_tokens_neox': 7281, 'n_words': 1781, 'path': 'Fast analysis.ipynb', 'repo_name': 'SandervWickeren/SpeedComparison', 'size': '19719'}