<commit_before>import sys
import os
from pynt import task
from tqdm import tqdm
import requests
import shutil

sys.path.append(".")

def rmdir_nonempty(dirname):
    for root, dirs, files in os.walk(dirname, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    try:
        os.rmdir(dirname)
    except OSError:
        print "Could not delete " + dirname

@task()
def download_wikidump():
    url = "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2"
    f = url.split("/")[-1]
    dest = "scraper/data"
    print "Downloading wikipedia dump to scraper/data"
    print "Downloading from {}".format(url)
    response = requests.get(url, stream=True)

    with open("{}/{}".format(dest, f), "wb") as handle:
        for data in tqdm(response.iter_content()):
            handle.write(data)

@task()
def start_server():
    destdir = "wikisonnet_api/bard/"

    # Copy the poem writing files
    rmdir_nonempty(destdir)
    os.mkdir(destdir)
    open(destdir + "__init__.py", 'a').close()
    shutil.copytree("db", destdir + "db")
    shutil.copytree("util", destdir + "util")
    shutil.copytree("wikibard", destdir + "wikibard")

    # Run the server
    from wikisonnet_api import application
    application.run()<commit_msg>Changes to build.py so we can start the server from outside of the directory<commit_after>import sys
import os
from pynt import task
from tqdm import tqdm
import requests
import shutil


srcdir = "/".join(os.path.realpath(__file__).split("/")[:-1]) + "/"
sys.path.append(srcdir)

def rmdir_nonempty(dirname):
    for root, dirs, files in os.walk(dirname, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    try:
        os.rmdir(dirname)
    except OSError:
        print "Could not delete " + dirname

@task()
def download_wikidump():
    url = "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2"
    f = url.split("/")[-1]
    dest = "scraper/data"
    print "Downloading wikipedia dump to scraper/data"
    print "Downloading from {}".format(url)
    response = requests.get(url, stream=True)

    with open("{}/{}".format(dest, f), "wb") as handle:
        for data in tqdm(response.iter_content()):
            handle.write(data)

@task()
def start_server():
    destdir = srcdir + "wikisonnet_api/bard/"

    # Copy the poem writing files
    rmdir_nonempty(destdir)
    os.mkdir(destdir)
    open(destdir + "__init__.py", 'a').close()
    shutil.copytree(srcdir + "db", destdir + "db")
    shutil.copytree(srcdir + "util", destdir + "util")
    shutil.copytree(srcdir + "wikibard", destdir + "wikibard")

    # Run the server
    from wikisonnet_api import application
    application.run()
{'fraction_non_alphanumeric': 0.1362856162032269, 'fraction_numerical': 0.0017164435290078957, 'mean_word_length': 3.1392045454545454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7167612', 'n_tokens_mistral': 998, 'n_tokens_neox': 992, 'n_words': 242}