<commit_before>#!/usr/bin/env python
# buildit 0.2 documentation

import os
import sys
import shutil

from glob import glob

import markdown2

def generate_docfile(file_name):
    txt_file = file_name
    name = file_name
    file_name = file_name.split('.')
    file_name.pop()
    name = name.split('/')
    name = name.pop()
    name = name.split('.')
    name = name.pop(0)
    file_name.append('html')
    file_name = '.'.join(file_name)
    file_name = '../html/{0}'.format(file_name)
    write_dir = file_name.split('/')
    write_dir.pop()
    write_dir = '/'.join(write_dir)
    try: os.makedirs(write_dir)
    except: pass
    html = markdown2.Markdown()
    file = open(file_name, 'w')
    header = open('templates/header.txt')
    for line in header:
        if '<title>' in line and '</title>' in line:
            line = line.format(name)
        file.write(line)
    header.close()
    txt_file = open(txt_file)
    for line in txt_file:
        file.write(html.convert(line))
    txt_file.close()
    footer = open('templates/footer.txt')
    for line in footer:
        file.write(line)
    footer.close()
    file.close()

def fix_paths(file_list):
    if sys.platform == 'win32':
        file_list = [file_name.replace('\\', '/') for file_name in file_list]
        file_list.sort()
    return file_list

if __name__ == '__main__':
    try:
        os.makedirs('../html')
    except OSError:
        pass
    file_list = []
    for root, dir, files in os.walk('.'):
        for file in files:<commit_msg>Going to be switching from markdown to textile for the documentation syntax<commit_after>#!/usr/bin/env python
# buildit 0.2 documentation

import os
import sys
import shutil

from glob import glob

import textile

def generate_docfile(file_name):
    txt_file = file_name
    name = file_name
    file_name = file_name.split('.')
    file_name.pop()
    name = name.split('/')
    name = name.pop()
    name = name.split('.')
    name = name.pop(0)
    file_name.append('html')
    file_name = '.'.join(file_name)
    file_name = '../html/{0}'.format(file_name)
    write_dir = file_name.split('/')
    write_dir.pop()
    write_dir = '/'.join(write_dir)
    try: os.makedirs(write_dir)
    except: pass
    file = open(file_name, 'w')
    header = open('templates/header.txt')
    for line in header:
        if '<title>' in line and '</title>' in line:
            line = line.format(name)
        file.write(line)
    header.close()
    txt_file = open(txt_file)
    for line in txt_file:
        file.write(textile.textile(line))
    txt_file.close()
    footer = open('templates/footer.txt')
    for line in footer:
        file.write(line)
    footer.close()
    file.close()

def fix_paths(file_list):
    if sys.platform == 'win32':
        file_list = [file_name.replace('\\', '/') for file_name in file_list]
        file_list.sort()
    return file_list

if __name__ == '__main__':
    try:
        os.makedirs('../html')
    except OSError:
        pass
    file_list = []
    for root, dir, files in os.walk('.'):
        for file in files:
{'fraction_non_alphanumeric': 0.13844653883652908, 'fraction_numerical': 0.004549886252843679, 'mean_word_length': 2.612676056338028, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7095558', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1043, 'n_words': 250}