<jupyter_start><jupyter_text>fastdoc.clean> Create "clean" version of notebooks with all outputs and non-header markdown removed<jupyter_code>#export
from fastdoc.imports import *
from fastdoc.asciidoc import copy_images
#export
def count_tk(fname):
    nb = read_nb(Path(fname))
    c = 0
    for cell in nb['cells']: c += len(re.findall('TK', cell['source']))
    return c
#export
def find_tokens(path='book'):
    path = Path(path)
    tks = [(f,count_tk(f)) for f in path.iterdir() if f.suffix == '.ipynb' and not f.name.startswith('_')]
    tks.sort(key=lambda o:o[1], reverse=True)
    if tks[0]==0: print("No TK remaining!")
    else:
        for f,tk in tks: 
            if tk !=0: print(f'{f} still has {tk} TK.')
# find_tokens()
#export
_re_header = re.compile(r'^#+\s+\S+')
_re_clean  = re.compile(r'^\s*#\s*clean\s*')
#export
def is_header_cell(cell): return _re_header.search(cell['source']) is not None
#export
def is_clean_cell(cell): return _re_clean.search(cell['source']) is not None
#export
_re_questionnaire = re.compile(r'^#+\s+Questionnaire')
#export
def get_stop_idx(cells):
    i = 0
    while i < len(cells) and _re_questionnaire.search(cells[i]['source']) is None: i+=1
    return i
#export
def clean_tags(cell):
    if is_header_cell(cell): return cell
    for attr in ["id", "caption", "alt", "width", "hide_input", "hide_output", "clean"]:
        cell["source"] = re.sub(r'#\s*' + attr + r'.*?($|\n)', '', cell["source"])
    return cell
#export
def clean_nb(fname, dest=None):
    fname = Path(fname)
    nb = read_nb(fname)
    i = get_stop_idx(nb['cells'])
    nb['cells'] = [clean_tags(c) for j,c in enumerate(nb['cells']) if 
                   c['cell_type']=='code' or is_header_cell(c) or is_clean_cell(c) or j >= i]
    if dest is None: dest = fname.parent/f'{fname.stem}_clean.ipynb'
    with open(dest, 'w') as f: nbformat.write(nb, f, version=4)
# clean_nb('test/_test.ipynb')
#export
def clean_all(path='book', dest_path=None):
    path = Path(path)
    dest_path = Path('..')/'fastbook' if dest_path is None else Path(dest_path)
    nbs = [f for f in path.iterdir() if f.suffix == '.ipynb' and not f.name.startswith('_')]
    for nb in nbs: 
        shutil.copy(nb, dest_path/nb.name)
        clean_nb(nb, dest=dest_path/'clean'/nb.name)
    shutil.copy(path/'utils.py', dest_path/'utils.py')
    copy_images(path, dest_path)<jupyter_output><empty_output><jupyter_text>Export -<jupyter_code>from nbdev.export import *
notebook2script()<jupyter_output>Converted 01_asciidoc.ipynb.
Converted 02_clean.ipynb.
{'fraction_non_alphanumeric': 0.16823529411764707, 'fraction_numerical': 0.005098039215686275, 'mean_word_length': 4.238193018480493, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '369265', 'license': 'Apache-2.0', 'n_tokens_mistral': 1005, 'n_tokens_neox': 974, 'n_words': 223, 'path': '01_clean.ipynb', 'repo_name': 'iyaja/fastdoc', 'size': '2550'}