<commit_before>import os
import os.path
import sys
import mash.rst
import shutil
import re
import mash.links
import simplejson as json

def mkdir_p(path):
    if not os.path.isdir(path):
        os.makedirs(path)

def generate(template_path, source_dir, static_dir, target_dir):
    shutil.copytree(static_dir, target_dir)
    
    template_file = open(template_path)
    template = template_file.read()
    template_file.close()

    for root, dirs, filenames in os.walk(source_dir):
        for filename in filenames:
            source_file = open(os.path.join(root, filename), "r")
            source = source_file.read()
            source_file.close()
            
            if filename.endswith(".html"):
                contents = source
            elif filename.endswith(".rst"):
                contents = mash.rst.rst_to_html_fragment(source)
            else:
                raise RuntimeError("Unrecognised file type: %s" % (filename,))
            full_contents = template.replace("<!-- CONTENT -->", contents)
            nav_placeholder = "<!-- NAVIGATION -->"
            if nav_placeholder in full_contents:
                nav_tree = json.loads(open("navigation.links").read())
                # FIXME: shouldn't use os.path.join since it depends on the OS
                path = os.path.join(root[len(source_dir)+1:], filename)
                mash.links.prune(nav_tree, path)
                nav_str = mash.links.tree_to_html(nav_tree)
                full_contents = full_contents.replace(nav_placeholder, nav_str)
            target_path = os.path.join(target_dir, root[len(source_dir)+1:], re.sub(r".[^.]+$", ".html", filename))
            print target_path
            mkdir_p(os.path.dirname(target_path))
            target = open(target_path, "w")
            target.write(full_contents)
            target.close()
        

<commit_msg>Generate current path properly<commit_after>import os
import os.path
import sys
import mash.rst
import shutil
import re
import mash.links
import simplejson as json

def mkdir_p(path):
    if not os.path.isdir(path):
        os.makedirs(path)

def generate(template_path, source_dir, static_dir, target_dir):
    shutil.copytree(static_dir, target_dir)
    
    template_file = open(template_path)
    template = template_file.read()
    template_file.close()

    for root, dirs, filenames in os.walk(source_dir):
        for filename in filenames:
            source_file = open(os.path.join(root, filename), "r")
            source = source_file.read()
            source_file.close()
            
            if filename.endswith(".html"):
                contents = source
            elif filename.endswith(".rst"):
                contents = mash.rst.rst_to_html_fragment(source)
            else:
                raise RuntimeError("Unrecognised file type: %s" % (filename,))
            full_contents = template.replace("<!-- CONTENT -->", contents)
            
            path_components = root[len(source_dir)+1:], re.sub(r".[^.]+$", ".html", filename)
            
            nav_placeholder = "<!-- NAVIGATION -->"
            if nav_placeholder in full_contents:
                nav_tree = json.loads(open("navigation.links").read())
                # FIXME: shouldn't use os.path.join since it depends on the OS
                mash.links.prune(nav_tree, os.path.join("/", *path_components))
                nav_str = mash.links.tree_to_html(nav_tree)
                full_contents = full_contents.replace(nav_placeholder, nav_str)
            target_path = os.path.join(target_dir, *path_components)
            print target_path
            mkdir_p(os.path.dirname(target_path))
            target = open(target_path, "w")
            target.write(full_contents)
            target.close()
        


{'fraction_non_alphanumeric': 0.11058326735286356, 'fraction_numerical': 0.000791765637371338, 'mean_word_length': 2.19831223628692, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5663303', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1082, 'n_words': 231}