<commit_before>import config 
import process_source as ps
import os
import shutil
import re,time

def build_site():
    if os.path.exists(config.site):
        shutil.rmtree(config.site)
    os.mkdir(config.site)
    shutil.copytree(config.static,config.site+"/"+config.static_dest)

    valid_extensions = config.extensions.keys()
    valid_extensions.append('html')
    for (dir,subs,files) in os.walk(config.content):
        dest = re.sub(config.content,config.site,dir)
        for sub in subs:
            if not os.path.exists(dest+"/"+sub):
                os.mkdir(dest+"/"+sub)
        for f in files:
            (name,ext)=os.path.splitext(f)
            if name[0] != '.' and ext[1:] in valid_extensions:
                result = ps.process_file(dir+"/"+f)
                outfilename=dest+"/"+name+".html"
                outfile = open(outfilename,"w")
                outfile.write(result)
                outfile.close()



def serve():
    build_site()

if __name__=="__main__":
    config.base_dir=os.getcwd()
    print "HELLO"
    serve()


<commit_msg>Now rebuilds site on changes (first cut)<commit_after>import config
import process_source as ps
import os,sys
import shutil
import re,time

def build_site():
    if os.path.exists(config.site):
        shutil.rmtree(config.site)
    os.mkdir(config.site)
    if os.path.exist(config.static):
        shutil.copytree(config.static,config.site+"/"+config.static_dest)

    valid_extensions = config.extensions.keys()
    valid_extensions.append('html')
    for (dir,subs,files) in os.walk(config.content):
        dest = re.sub(config.content,config.site,dir)
        for sub in subs:
            if not os.path.exists(dest+"/"+sub):
                os.mkdir(dest+"/"+sub)
        for f in files:
            (name,ext)=os.path.splitext(f)
            if name[0] != '.' and ext[1:] in valid_extensions:
                result = ps.process_file(dir+"/"+f)
                outfilename=dest+"/"+name+".html"
                outfile = open(outfilename,"w")
                outfile.write(result)
                outfile.close()


def serve():
    olds=" ".join([" ".join(["%f"%os.stat(dir+"/"+f).st_mtime for f in files if f[0]!='.']) for (dir,subs,files) in os.walk(config.content)])
    while  True:
        time.sleep(1)
        news=" ".join([" ".join(["%f"%os.stat(dir+"/"+f).st_mtime for f in files if f[0] !='.']) for (dir,subs,files) in os.walk(config.content)])
        if news!=olds:
            print "REBUILDING"
            build_site()
            olds = news



if __name__=="__main__":
    if len(sys.argv)>1:
        os.chdir(sys.argv[-1])
        config.base_dir=os.getcwd()
        build_site()
    if sys.argv[1]=='serve':
        serve()
{'fraction_non_alphanumeric': 0.15528634361233482, 'fraction_numerical': 0.003671071953010279, 'mean_word_length': 2.3683559950556243, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '169810', 'n_tokens_mistral': 929, 'n_tokens_neox': 902, 'n_words': 173}