<commit_before>"""Tasks related to projects, including fetching repository code, cleaning
``conf.py`` files, and rebuilding documentation.
"""
import decimal
import fnmatch
import os
import re
import shutil
import json
import logging
import operator

from celery.decorators import task
from django.db import transaction
from django.conf import settings
import redis
from sphinx.ext import intersphinx
import slumber


from builds.models import Version, Build
from doc_builder import loading as builder_loading
from doc_builder.base import restoring_chdir
from projects.exceptions import ProjectImportError
from projects.models import ImportedFile, Project
from projects.utils import (
    DictObj,
    mkversion,
    purge_version,
    run,
    safe_write,
    slugify_uniquely,
    )
from tastyapi import client as tastyapi_client
from tastyapi import api
from core.utils import copy_to_app_servers, run_on_app_servers

ghetto_hack = re.compile(
    r'(?P<key>.*)\s*=\s*u?\[?[\'\"](?P<value>.*)[\'\"]\]?')

log = logging.getLogger(__name__)

@task
def remove_dir(path):
    """
    Remove a directory on the build/celery server.

    This is mainly a wrapper around shutil.rmtree so that app servers
    can kill things on the build server.
    """
    log.info("Removing %s" % path)
    shutil.rmtree(path)


@task
@restoring_chdir
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    update_output = kwargs.get('update_output', {})
    project_data = api.project(pk).get()
    del project_data['users']
    del project_data['resource_uri']
    del project_data['absolute_url']
    project = Project(**project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0
    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
        del version_data['resource_uri']
    else:
        #Create or use the 'latest' branch, which is the default for a project.
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            version_data = api.version(project.slug).get(slug='latest')['objects'][0]
            del version_data['resource_uri']
        except (slumber.exceptions.HttpClientError, IndexError) as exc:
            #if exc.response.status_code in [404,500]:
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
                )
            try:
                version_data = api.version.post(version_data)
                del version_data['resource_uri']
            except Exception as e:
                raise e
    version_data['project'] = project
    version = Version(**version_data)
    version.save = new_save

    if not version_pk:
        #Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch<commit_msg>Don't bomb on exception updating version.<commit_after>"""Tasks related to projects, including fetching repository code, cleaning
``conf.py`` files, and rebuilding documentation.
"""
import decimal
import fnmatch
import os
import re
import shutil
import json
import logging
import operator

from celery.decorators import task
from django.db import transaction
from django.conf import settings
import redis
from sphinx.ext import intersphinx
import slumber


from builds.models import Version, Build
from doc_builder import loading as builder_loading
from doc_builder.base import restoring_chdir
from projects.exceptions import ProjectImportError
from projects.models import ImportedFile, Project
from projects.utils import (
    DictObj,
    mkversion,
    purge_version,
    run,
    safe_write,
    slugify_uniquely,
    )
from tastyapi import client as tastyapi_client
from tastyapi import api
from core.utils import copy_to_app_servers, run_on_app_servers

ghetto_hack = re.compile(
    r'(?P<key>.*)\s*=\s*u?\[?[\'\"](?P<value>.*)[\'\"]\]?')

log = logging.getLogger(__name__)

@task
def remove_dir(path):
    """
    Remove a directory on the build/celery server.

    This is mainly a wrapper around shutil.rmtree so that app servers
    can kill things on the build server.
    """
    log.info("Removing %s" % path)
    shutil.rmtree(path)


@task
@restoring_chdir
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    update_output = kwargs.get('update_output', {})
    project_data = api.project(pk).get()
    del project_data['users']
    del project_data['resource_uri']
    del project_data['absolute_url']
    project = Project(**project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0
    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
        del version_data['resource_uri']
    else:
        #Create or use the 'latest' branch, which is the default for a project.
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            version_data = api.version(project.slug).get(slug='latest')['objects'][0]
            del version_data['resource_uri']
        except (slumber.exceptions.HttpClientError, IndexError) as exc:
            #if exc.response.status_code in [404,500]:
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
                )
            try:
                version_data = api.version.post(version_data)
                del version_data['resource_uri']
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                #raise e
    version_data['project'] = project
    version = Version(**version_data)
    version.save = new_save

    if not version_pk:
        #Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
{'fraction_non_alphanumeric': 0.08921239268383725, 'fraction_numerical': 0.0022396416573348264, 'mean_word_length': 2.747319347319347, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6061269', 'n_tokens_mistral': 2267, 'n_tokens_neox': 2202, 'n_words': 718}