<commit_before>import sys
import os, shutil, subprocess
from glob import glob

site_title = 'wabbo'
include_dir = 'includes/'
exclude_files = ['readme.md']
out_folder = '_out/'


fname_no_ext = lambda fn: fn[:fn.index('.')]

def pandocConvert(pathto, fname):
    dothtml = fname[:fname.index('.')] + '.html'
    in_file = pathto + fname
    out_file = out_folder + pathto + dothtml

    pandoc_call = ['pandoc', '-s', in_file, '-t', 'html5', '-o', out_file,
                          '--include-in-header', include_dir+'header.html',
                          '--include-before-body', include_dir+'cover.html',
                          '--include-after-body', include_dir+'footer.html',
                          '--mathjax', '--smart', '--title-prefix', site_title]

    p = subprocess.call(pandoc_call)

    return


def isPage(fname):
    pages = ['.md', '.rst']
    return ('.' in fname) \
        and (fname[fname.index('.'):] in pages) \
        and fname not in exclude_files



def convert_folder(folder, wl=None):
    if folder == '':
        folder = './' 

    if wl is not None:
        filter = lambda fn: fname_no_ext(fn) in wl
    else:
        filter = lambda fn: True

    out_path = out_folder + folder

    for fname in os.listdir(folder):
        if isPage(fname) and filter(fname):
            if not os.path.exists(out_path):
                os.makedirs(out_path)

            pandocConvert(folder, fname)
    

# Compilation script begins here
convert_folder('')
convert_folder('blov/')
convert_folder('notes/', wl=['linalg'])<commit_msg>add blacklist parameter to convert_folder. remove exclude_files check from isPage<commit_after>import sys
import os, shutil, subprocess
from glob import glob

site_title = 'wabbo'
include_dir = 'includes/'
out_folder = '_out/'


fname_no_ext = lambda fn: fn[:fn.index('.')]

def pandocConvert(pathto, fname):
    dothtml = fname[:fname.index('.')] + '.html'
    in_file = pathto + fname
    out_file = out_folder + pathto + dothtml

    pandoc_call = ['pandoc', '-s', in_file, '-t', 'html5', '-o', out_file,
                          '--include-in-header', include_dir+'header.html',
                          '--include-before-body', include_dir+'cover.html',
                          '--include-after-body', include_dir+'footer.html',
                          '--mathjax', '--smart', '--title-prefix', site_title]

    p = subprocess.call(pandoc_call)

    return


def isPage(fname):
    pages = ['.md', '.rst']
    return ('.' in fname) \
        and (fname[fname.index('.'):] in pages)


def convert_folder(folder, wl=None, bl=None):
    if folder == '':
        folder = './' 

    if wl is not None:
        filter = lambda fn: fname_no_ext(fn) in wl
    else:
        filter = lambda fn: True

    if bl is not None:
        filter = lambda fn: fname_no_ext(fn) not in bl
    else:
        filter = lambda fn: True


    out_path = out_folder + folder

    for fname in os.listdir(folder):
        if isPage(fname) and filter(fname):
            if not os.path.exists(out_path):
                os.makedirs(out_path)

            pandocConvert(folder, fname)
    

# Compilation script begins here
convert_folder('', bl=['readme'])
convert_folder('blov/')
convert_folder('notes/', wl=['linalg'])
{'fraction_non_alphanumeric': 0.14508723599632692, 'fraction_numerical': 0.0006121824303642485, 'mean_word_length': 2.397089397089397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2033189', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1084, 'n_words': 260}