<commit_before>from celery.decorators import task
from celery.task.schedules import crontab
from celery.decorators import periodic_task

from projects.constants import SCRAPE_CONF_SETTINGS
from projects.models import Project, Conf
from projects.utils import  find_file, run

import os
import re
import glob
import fnmatch


ghetto_hack = re.compile(r'(?P<key>.*)\s*=\s*u?[\'\"](?P<value>.*)[\'\"]')

@task
def update_docs(pk):
    project = Project.objects.get(pk=pk)

    path = project.user_doc_path
    if not os.path.exists(path):
        os.makedirs(path)

    if project.is_imported:
        update_imported_docs(project)
        scrape_conf_file(project)
    else:
        update_created_docs(project)

    # kick off a build
    build_docs(project)


def update_imported_docs(project):
    """
    A Celery task that updates the documentation for a project.
    """
    os.chdir(path)
    if os.path.exists(os.path.join(path, project.slug)):
        os.chdir(project.slug)
        if project.repo_type is 'hg':
            command = 'hg update -C -r . '
        else:
            command = 'git fetch && git reset --hard origin/master'
        print command
        run(command)
    else:
        repo = project.repo
        if project.repo_type is 'hg':
            command = 'hg clone %s %s' % (repo, project.slug)
        else:
            repo.replace('.git', '')
            command = 'git clone %s.git %s' % (repo, project.slug)
        print command
        run(command)


def scrape_conf_file(project):
    conf_dir = project.find('conf.py')[0].replace('/conf.py', '')
    os.chdir(conf_dir)<commit_msg>Small bugfix.<commit_after>from celery.decorators import task
from celery.task.schedules import crontab
from celery.decorators import periodic_task

from projects.constants import SCRAPE_CONF_SETTINGS
from projects.models import Project, Conf
from projects.utils import  find_file, run

import os
import re
import glob
import fnmatch


ghetto_hack = re.compile(r'(?P<key>.*)\s*=\s*u?[\'\"](?P<value>.*)[\'\"]')

@task
def update_docs(pk):
    project = Project.objects.get(pk=pk)

    path = project.user_doc_path
    if not os.path.exists(path):
        os.makedirs(path)

    if project.is_imported:
        update_imported_docs(project)
        scrape_conf_file(project)
    else:
        update_created_docs(project)

    # kick off a build
    build_docs(project)


def update_imported_docs(project):
    """
    A Celery task that updates the documentation for a project.
    """
    path = project.user_doc_path
    os.chdir(path)
    if os.path.exists(os.path.join(path, project.slug)):
        os.chdir(project.slug)
        if project.repo_type is 'hg':
            command = 'hg update -C -r . '
        else:
            command = 'git fetch && git reset --hard origin/master'
        print command
        run(command)
    else:
        repo = project.repo
        if project.repo_type is 'hg':
            command = 'hg clone %s %s' % (repo, project.slug)
        else:
            repo.replace('.git', '')
            command = 'git clone %s.git %s' % (repo, project.slug)
        print command
        run(command)


def scrape_conf_file(project):
    conf_dir = project.find('conf.py')[0].replace('/conf.py', '')
    os.chdir(conf_dir)
{'fraction_non_alphanumeric': 0.12075817792723938, 'fraction_numerical': 0.0006114338122898196, 'mean_word_length': 2.664053751399776, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5241045', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1048, 'n_words': 254}