<commit_before>import sys
import os


def find_code(text):
    START_TAG = '```lpg'
    END_TAG = '```'
    first_index = text.find(START_TAG)
    if first_index == -1:
        return None, None
    last_index = text.find(END_TAG, first_index + 1)
    return first_index + len(START_TAG), last_index


def process_file(path):
    content = open(path, 'r').read()
    left, right = find_code(content)
    code_pieces = []
    while left is not None:
        code_pieces.append(content[left:right].strip())
        content = content[right:]
        left, right = find_code(content)
    return code_pieces


def write_to_file(file_name, code):
    for i, code_block in enumerate(code):
        print_block = '// %s : %i\n%s\n' % (file_name, i, code_block)
        destination.write(print_block)


# Main Program
root_dir = '.'
destination = open('documentation.lpg', 'w')
if len(sys.argv) == 2:
    root_dir = sys.argv[1]

for root, dirs, files in os.walk(root_dir):
    for fi in files:
        if fi.endswith('md'):
            code = process_file(os.path.join(root, fi))
            write_to_file(fi, code)
destination.close()
<commit_msg>Split the documentation compilation into different files<commit_after>import sys
import os


def find_code(text):
    START_TAG = '```lpg'
    END_TAG = '```'
    first_index = text.find(START_TAG)
    if first_index == -1:
        return None, None
    last_index = text.find(END_TAG, first_index + 1)
    return first_index + len(START_TAG), last_index


def process_file(path):
    content = open(path, 'r').read()
    left, right = find_code(content)
    code_pieces = []
    while left is not None:
        code_pieces.append(content[left:right].strip())
        content = content[right:]
        left, right = find_code(content)
    return code_pieces


def write_to_file(file_name, code):
    with open(file_name, 'w') as f:
        f.write(code)


def write_codeblocks_to_different_files(code_blocks):
    for i, code_block in enumerate(code_blocks):
        write_to_file('codeblock%i.lpg' % i, code_block)

# Main Program
root_dir = '.'
if len(sys.argv) == 2:
    root_dir = sys.argv[1]

if not os.path.exists(root_dir):
    raise Exception('Directory does not exist')

for root, dirs, files in os.walk(root_dir):
    markdown_files = [f for f in files if f.endswith('md')]
    for fi in markdown_files:
        code = process_file(os.path.join(root, fi))
        write_codeblocks_to_different_files(code)

{'fraction_non_alphanumeric': 0.12342158859470469, 'fraction_numerical': 0.003258655804480652, 'mean_word_length': 3.278745644599303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945030', 'n_tokens_mistral': 864, 'n_tokens_neox': 829, 'n_words': 201}