<jupyter_start><jupyter_text>Import<jupyter_code># export
import re
import json
import time
import hashlib
from pathlib import Path

import numpy
import numba
import requests
import seaborn as sns

import ipykernel
import nbdev.export
from IPython.display import Javascript
from notebook.notebookapp import list_running_servers
from IPython.core.debugger import set_trace<jupyter_output><empty_output><jupyter_text>Tests `assert_allclose` checks if two things, `A` and `B`, are close to each other.NOTE: I'm assuming the format of the inputs is the same; if not I'm assuming this is programmer error.<jupyter_code># export
def assert_allclose(A, B, **kwargs):
    if isinstance(A, tuple):            
        for a,b in zip(A,B): assert_allclose(a, b, **kwargs) # Possibly add "strict" keyword here
    elif isinstance(A, dict):           
        for key in A.keys() | B.keys(): assert_allclose(A[key], B[key], **kwargs)
    else:
        try:    assert(numpy.allclose(A, B, **kwargs))
        except: assert(numpy.all(A == B))
A = numpy.random.normal((4,3))
B = numpy.random.normal(size=(4,3))
A, B
assert_allclose(A, A+1e-5, atol=1e-5)
assert_allclose((A, (B, {'test': 1.})), (A+1e-5, (B+1e-5, {'test': 1 + 1e-5})), atol=1e-5)<jupyter_output><empty_output><jupyter_text>Image processing<jupyter_code># export
def rgb2gray(arr): # From Pillow documentation
    return arr[:,:,0]*(299/1000) + arr[:,:,1]*(587/1000) + arr[:,:,2]*(114/1000)<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code># export
def get_colors(n): return sns.color_palette(None, n)<jupyter_output><empty_output><jupyter_text>Notebook stuff These are kind of hacky, but I like being able to rerun a notebook and have it auto save/build/convert at the end<jupyter_code># export
def get_notebook_file():
    id_kernel = re.search('kernel-(.*).json', ipykernel.connect.get_connection_file()).group(1)
    for server in list_running_servers():
        response = requests.get(requests.compat.urljoin(server['url'], 'api/sessions'),
                                params={'token': server.get('token', '')})
        for r in json.loads(response.text):
            if 'kernel' in r and r['kernel']['id'] == id_kernel:
                return Path(r['notebook']['path'])
assert_allclose(get_notebook_file().as_posix(), 'utils.ipynb')
# export
def save_notebook():
    file_notebook = get_notebook_file()
    _get_md5 = lambda : hashlib.md5(file_notebook.read_bytes()).hexdigest() 
    md5_start = _get_md5()
    display(Javascript('IPython.notebook.save_checkpoint();')) # Asynchronous
    while md5_start == _get_md5(): time.sleep(1e-1)
# export
def build_notebook(save=True):
    if save: save_notebook()
    nbdev.export.notebook2script(fname=get_notebook_file().as_posix())
# export
def convert_notebook(save=True, t='markdown'):
    if save: save_notebook()
    os.system(f'jupyter nbconvert --to {t} {get_notebook_file().as_posix()}')<jupyter_output><empty_output><jupyter_text>Build<jupyter_code>build_notebook()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14314049586776859, 'fraction_numerical': 0.017520661157024792, 'mean_word_length': 4.921722113502935, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '567397', 'license': 'Apache-2.0', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1067, 'n_words': 234, 'path': 'utils.ipynb', 'repo_name': 'justinblaber/cs371', 'size': '3025'}