<jupyter_start><jupyter_text>1. Installation<jupyter_code>import sys
!{sys.executable} -m pip uninstall -y glcr
!{sys.executable} -m pip install '..'<jupyter_output>Uninstalling glcr-1.1:
  Successfully uninstalled glcr-1.1
[33mYou are using pip version 18.0, however version 18.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.[0m
Processing /home/giuseppe/git/ddna/lcs_module
Building wheels for collected packages: glcr
  Running setup.py bdist_wheel for glcr ... [?25ldone
[?25h  Stored in directory: /tmp/pip-ephem-wheel-cache-0yc6vk43/wheels/49/b9/38/953ac34c264e3aba4a952ede71c58958f369de1794e3a3ce1d
Successfully built glcr
Installing collected packages: glcr
Successfully installed glcr-1.1
[33mYou are using pip version 18.0, however version 18.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.[0m<jupyter_text>--- 2. Imports and utils<jupyter_code>import numpy as np
import glcr
import os, re

def unicode_to_ascii(array):
    def f(item):
        return str(item)+'\0'
    def v(x):
        return np.vectorize(f)(x)
    return v(array).astype('S')

def purge(dir, pattern):
    for f in os.listdir(dir):
        if re.search(pattern+'*', f):
            os.remove(os.path.join(dir, f))
            
class verbosity:
    TEST = 0           # No result produced
    IN_MEMORY = 1      # Returns the columns lcs and num_words
    FILE = 2           # Returns 0 and saves a .mat file (lcs, num_words, begin, end) and a .gsa (i, wordindex)
    FILE_EXTENDED = 3  # Returns 0 and saves a .mat file (lcs, num_words, begin, end, sequence) and a .gsa (i, wordindex)

dir = '/tmp'
file = 'glcr_cache'
out_file = dir + '/' + file<jupyter_output><empty_output><jupyter_text>--- 3. Compute LCS From numpy array<jupyter_code>purge(dir, file)

alphabet = [chr(i) for i in range(48, 51)]
np_alphabet = np.array(alphabet, dtype="|S1")
length = 1000
array = np.random.choice(np_alphabet, [length, 1000])
array = [b"".join(array[i]).decode('utf-8') for i in range(length)]
array = unicode_to_ascii(array)

glcr.longest_common_subsequence(array, "", out_file, verbosity.FILE_EXTENDED)

from multiprocessing import Process
from glcr import longest_common_subsequence

p = Process(target=longest_common_subsequence, args=(array, '', '/tmp/glcr_cache', verbosity.FILE))
p.start()
p.join()<jupyter_output><empty_output><jupyter_text>From file<jupyter_code>purge(dir, file)

glcr.longest_common_subsequence(None, "input_4000.txt", '/tmp/glcr_cache', verbosity.IN_MEMORY)<jupyter_output><empty_output><jupyter_text>--- 4. Benchmark LCS module<jupyter_code>import glcr
import pandas as pd
import numpy as np
import timeit

SEQ_LEN = range(1000, 4000, 1000)
NO_SEQ = range(1000, 4000, 1000)
alphabet = [chr(i) for i in range(48,51)]
np_alphabet = np.array(alphabet, dtype="|S1")

cols = ['sequence_len', 'no_sequences', 'ex_time']
res = pd.DataFrame(columns=cols)
for l in SEQ_LEN:
    for n in NO_SEQ:
        codes = np.random.choice(np_alphabet, [l, n])
        codes = [b"".join(codes[i]).decode('utf-8') for i in range(l)]
        codes = unicode_to_ascii(codes)
        t = timeit.timeit('glcr.longest_common_subsequence(codes, "", "/tmp/glcr_cache_arg", verbosity.TEST)',
                          setup='import glcr', number=1, globals=globals())
        res = res.append(pd.DataFrame([[l, n, t]], columns=cols), ignore_index=True)
print(res)
%matplotlib inline
from matplotlib import pyplot as plt
plt.rcParams['figure.figsize'] = [12, 8]

glen = res.groupby('no_sequences')
for g, df in glen:
    plt.plot(df.sequence_len, df.ex_time, label='no_sequence='+str(g))
    plt.xlabel('sequence_len')
    plt.ylabel('execution time (s)')
plt.legend()
plt.show()

glen = res.groupby('sequence_len')
for g, df in glen:
    plt.plot(df.no_sequences, df.ex_time, label='sequence_len='+str(g))
    plt.xlabel('no_sequences')
    plt.ylabel('execution time (s)')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13569099573614246, 'fraction_numerical': 0.03260596940055179, 'mean_word_length': 4.418478260869565, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '273387', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1564, 'n_tokens_neox': 1443, 'n_words': 357, 'path': 'notebooks/example-usage.ipynb', 'repo_name': 'WAFI-CNR/glcr', 'size': '3987'}