<commit_before>"""
Tasks related to projects.

This includes fetching repository code, cleaning ``conf.py`` files, and
rebuilding documentation.
"""

from __future__ import absolute_import

import datetime
import hashlib
import json
import logging
import os
import shutil
import socket
from collections import defaultdict

import requests
from builtins import str
from celery import Task
from celery.exceptions import SoftTimeLimitExceeded
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from readthedocs_build.config import ConfigError
from slumber.exceptions import HttpClientError

from .constants import LOG_TEMPLATE
from .exceptions import RepositoryError
from .models import ImportedFile, Project, Domain
from .signals import before_vcs, after_vcs, before_build, after_build
from readthedocs.builds.constants import (LATEST,
                                          BUILD_STATE_CLONING,
                                          BUILD_STATE_INSTALLING,
                                          BUILD_STATE_BUILDING,
                                          BUILD_STATE_FINISHED)
from readthedocs.builds.models import Build, Version, APIVersion
from readthedocs.builds.signals import build_complete
from readthedocs.builds.syncers import Syncer
from readthedocs.cdn.purge import purge
from readthedocs.core.resolver import resolve_path
from readthedocs.core.symlink import PublicSymlink, PrivateSymlink
from readthedocs.core.utils import send_email, broadcast
from readthedocs.doc_builder.config import load_yaml_config
from readthedocs.doc_builder.constants import DOCKER_LIMITS
from readthedocs.doc_builder.environments import (LocalEnvironment,
                                                  DockerEnvironment)
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.doc_builder.python_environments import Virtualenv, Conda
from readthedocs.projects.models import APIProject
from readthedocs.restapi.client import api as api_v2
from readthedocs.restapi.utils import index_search_request
from readthedocs.search.parse_json import process_all_json_files
from readthedocs.search.utils import process_mkdocs_json
from readthedocs.vcs_support import utils as vcs_support_utils
from readthedocs.worker import app


log = logging.getLogger(__name__)

HTML_ONLY = getattr(settings, 'HTML_ONLY_PROJECTS', ())


class UpdateDocsTask(Task):

    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.
    """

    max_retries = 5
    default_retry_delay = (7 * 60)
    name = __name__ + '.update_docs'

    def __init__(self, build_env=None, python_env=None, config=None,
                 force=False, search=True, localmedia=True,
                 build=None, project=None, version=None):
        self.build_env = build_env
        self.python_env = python_env
        self.build_force = force
        self.build_search = search
        self.build_localmedia = localmedia
        self.build = {}
        if build is not None:
            self.build = build
        self.version = {}
        if version is not None:
            self.version = version
        self.project = {}
        if project is not None:
            self.project = project
        if config is not None:
            self.config = config

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    # pylint: disable=arguments-differ
    def run(self, pk, version_pk=None, build_pk=None, record=True,
            docker=False, search=True, force=False, localmedia=True, **__):
        """
        Run a documentation build.

        This is fully wrapped in exception handling to account for a number of
        failure cases. We first run a few commands in a local build environment,
        but do not report on environment success. This avoids a flicker on the
        build output page where the build is marked as finished in between the
        local environment steps and the docker build steps.

        If a failure is raised, or the build is not successful, return ``False``,
        otherwise, ``True``.

        Unhandled exceptions raise a generic user facing error, which directs
        the user to bug us. It is therefore a benefit to have as few unhandled
        errors as possible.

        :param pk int: Project id
        :param version_pk int: Project Version id
        :param build_pk int: Build id
        :param record bool: record a build object in the database
        :param docker bool: use docker to build the project
        :param search bool: update search
        :param force bool: force Sphinx build
        :param localmedia: update localmedia
        :returns: if build was successful or not
        :rtype: bool
        """
        try:
            self.project = self.get_project(pk)
            self.version = self.get_version(self.project, version_pk)
            self.build = self.get_build(build_pk)
            self.build_search = search
            self.build_localmedia = localmedia
            self.build_force = force
            self.config = None

            setup_successful = self.run_setup(record=record)
            if not setup_successful:
                return False
        # Catch unhandled errors in the setup step
        except Exception as e:  # noqa
            log.exception(
                'An unhandled exception was raised during build setup',
                extra={'tags': {'build': build_pk}}
            )
            self.setup_env.failure = BuildEnvironmentError(
                BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                    build_id=build_pk,
                )
            )
            self.setup_env.update_build(BUILD_STATE_FINISHED)
            return False
        else:
            # No exceptions in the setup step, catch unhandled errors in the
            # build steps
            try:
                self.run_build(record=record, docker=docker)
            except Exception as e:  # noqa
                log.exception(
                    'An unhandled exception was raised during project build',
                    extra={'tags': {'build': build_pk}}
                )
                self.build_env.failure = BuildEnvironmentError(
                    BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                        build_id=build_pk,
                    )
                )
                self.build_env.update_build(BUILD_STATE_FINISHED)
                return False

        return True

    def run_setup(self, record=True):
        """
        Run setup in the local environment.

        Return True if successful.
        """
        self.setup_env = LocalEnvironment(
            project=self.project,
            version=self.version,
            build=self.build,
            record=record,
            update_on_success=False,
        )

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.',
                    status_code=423
                )

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e))
                )

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' % self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
                self.send_notifications()

            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True

    def run_build(self, docker=False, record=True):
        """
        Build the docs in an environment.

        If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
        setting, then build in a Docker environment. Otherwise build locally.
        """
        env_vars = self.get_env_vars()

        if docker or settings.DOCKER_ENABLE:
            env_cls = DockerEnvironment
        else:
            env_cls = LocalEnvironment
        self.build_env = env_cls(project=self.project, version=self.version, config=self.config,
                                 build=self.build, record=record, environment=env_vars)

        # Environment used for building code, usually with Docker
        with self.build_env:

            if self.project.documentation_type == 'auto':
                self.update_documentation_type()

            python_env_cls = Virtualenv
            if self.config.use_conda:
                self._log('Using conda')
                python_env_cls = Conda
            self.python_env = python_env_cls(version=self.version,
                                             build_env=self.build_env,
                                             config=self.config)

            try:
                self.setup_environment()

                # TODO the build object should have an idea of these states, extend
                # the model to include an idea of these outcomes
                outcomes = self.build_docs()
                build_id = self.build.get('id')
            except SoftTimeLimitExceeded:
                raise BuildEnvironmentError(_('Build exited due to time out'))

            # Finalize build and update web servers
            if build_id:
                self.update_app_instances(
                    html=bool(outcomes['html']),
                    search=bool(outcomes['search']),
                    localmedia=bool(outcomes['localmedia']),
                    pdf=bool(outcomes['pdf']),
                    epub=bool(outcomes['epub']),
                )
            else:
                log.warning('No build ID, not syncing files')

        if self.build_env.failed:
            self.send_notifications()

        build_complete.send(sender=Build, build=self.build_env.build)

    @staticmethod
    def get_project(project_pk):
        """Get project from API."""
        project_data = api_v2.project(project_pk).get()
        return APIProject(**project_data)

    @staticmethod
    def get_version(project, version_pk):
        """Ensure we're using a sane version."""
        if version_pk:
            version_data = api_v2.version(version_pk).get()
        else:
            version_data = (api_v2
                            .version(project.slug)
                            .get(slug=LATEST)['objects'][0])
        return APIVersion(**version_data)

    @staticmethod
    def get_build(build_pk):
        """
        Retrieve build object from API.

        :param build_pk: Build primary key
        """
        build = {}
        if build_pk:
            build = api_v2.build(build_pk).get()
        return dict((key, val) for (key, val) in list(build.items())
                    if key not in ['project', 'version', 'resource_uri',
                                   'absolute_uri'])

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        update_imported_docs(self.version.pk)
        commit = self.project.vcs_repo(self.version.slug).commit
        if commit:
            self.build['commit'] = commit

    def get_env_vars(self):
        """Get bash environment variables used for all builder commands."""
        env = {
            'READTHEDOCS': True,
            'READTHEDOCS_VERSION': self.version.slug,
            'READTHEDOCS_PROJECT': self.project.slug
        }

        if self.config.use_conda:
            env.update({
                'CONDA_ENVS_PATH': os.path.join(self.project.doc_path, 'conda'),
                'CONDA_DEFAULT_ENV': self.version.slug,
                'BIN_PATH': os.path.join(self.project.doc_path, 'conda', self.version.slug, 'bin')
            })
        else:
            env.update({
                'BIN_PATH': os.path.join(self.project.doc_path, 'envs', self.version.slug, 'bin')
            })

        return env

    def set_valid_clone(self):
        """Mark on the project that it has been cloned properly."""
        project_data = api_v2.project(self.project.pk).get()
        project_data['has_valid_clone'] = True
        api_v2.project(self.project.pk).put(project_data)
        self.project.has_valid_clone = True

    def update_documentation_type(self):
        """
        Force Sphinx for 'auto' documentation type.

        This used to determine the type and automatically set the documentation
        type to Sphinx for rST and Mkdocs for markdown. It now just forces
        Sphinx, due to markdown support.
        """
        ret = 'sphinx'
        project_data = api_v2.project(self.project.pk).get()
        project_data['documentation_type'] = ret
        api_v2.project(self.project.pk).put(project_data)
        self.project.documentation_type = ret

    def update_app_instances(self, html=False, localmedia=False, search=False,
                             pdf=False, epub=False):
        """
        Update application instances with build artifacts.

        This triggers updates across application instances for html, pdf, epub,
        downloads, and search. Tasks are broadcast to all web servers from here.
        """
        # Update version if we have successfully built HTML output
        try:
            if html:
                version = api_v2.version(self.version.pk)
                version.patch({
                    'active': True,
                    'built': True,
                })
        except HttpClientError:
            log.exception('Updating version failed, skipping file sync: version=%s' % self.version)

        # Broadcast finalization steps to web application instances
        broadcast(
            type='app',
            task=sync_files,
            args=[
                self.project.pk,
                self.version.pk,
            ],
            kwargs=dict(
                hostname=socket.gethostname(),
                html=html,
                localmedia=localmedia,
                search=search,
                pdf=pdf,
                epub=epub,
            ),
            callback=sync_callback.s(version_pk=self.version.pk, commit=self.build['commit']),
        )

    def setup_environment(self):
        """
        Build the virtualenv and install the project into it.

        Always build projects with a virtualenv.

        :param build_env: Build environment to pass commands and execution through.
        """
        self.build_env.update_build(state=BUILD_STATE_INSTALLING)

        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):

            # Check if the python version/build image in the current venv is the
            # same to be used in this build and if it differs, wipe the venv to
            # avoid conflicts.
            if self.python_env.is_obsolete:
                self.python_env.delete_existing_venv_dir()
            else:
                self.python_env.delete_existing_build_dir()

            self.python_env.setup_base()
            self.python_env.save_environment_json()
            self.python_env.install_core_requirements()
            self.python_env.install_user_requirements()
            self.python_env.install_package()

    def build_docs(self):
        """
        Wrapper to all build functions.

        Executes the necessary builds for this task and returns whether the
        build was successful or not.

        :returns: Build outcomes with keys for html, search, localmedia, pdf,
                  and epub
        :rtype: dict
        """
        self.build_env.update_build(state=BUILD_STATE_BUILDING)
        before_build.send(sender=self.version)

        outcomes = defaultdict(lambda: False)
        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
            outcomes['html'] = self.build_docs_html()
            outcomes['search'] = self.build_docs_search()
            outcomes['localmedia'] = self.build_docs_localmedia()
            outcomes['pdf'] = self.build_docs_pdf()
            outcomes['epub'] = self.build_docs_epub()

        after_build.send(sender=self.version)
        return outcomes

    def build_docs_html(self):
        """Build HTML docs."""
        html_builder = get_builder_class(self.project.documentation_type)(
            build_env=self.build_env,
            python_env=self.python_env,
        )
        if self.build_force:
            html_builder.force()
        html_builder.append_conf()
        success = html_builder.build()
        if success:
            html_builder.move()

        # Gracefully attempt to move files via task on web workers.
        try:
            broadcast(type='app', task=move_files,
                      args=[self.version.pk, socket.gethostname()],
                      kwargs=dict(html=True)
                      )
        except socket.error:
            log.exception('move_files task has failed on socket error.')

        return success

    def build_docs_search(self):
        """Build search data with separate build."""
        if self.build_search:
            if self.project.is_type_mkdocs:
                return self.build_docs_class('mkdocs_json')
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_search')
        return False

    def build_docs_localmedia(self):
        """Get local media files with separate build."""
        if 'htmlzip' not in self.config.formats:
            return False

        if self.build_localmedia:
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_singlehtmllocalmedia')
        return False

    def build_docs_pdf(self):
        """Build PDF docs."""
        if ('pdf' not in self.config.formats or
            self.project.slug in HTML_ONLY or<commit_msg>Remove mkdocs indexing<commit_after>"""
Tasks related to projects.

This includes fetching repository code, cleaning ``conf.py`` files, and
rebuilding documentation.
"""

from __future__ import absolute_import

import datetime
import hashlib
import json
import logging
import os
import shutil
import socket
from collections import defaultdict

import requests
from builtins import str
from celery import Task
from celery.exceptions import SoftTimeLimitExceeded
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from readthedocs_build.config import ConfigError
from slumber.exceptions import HttpClientError

from .constants import LOG_TEMPLATE
from .exceptions import RepositoryError
from .models import ImportedFile, Project, Domain
from .signals import before_vcs, after_vcs, before_build, after_build
from readthedocs.builds.constants import (LATEST,
                                          BUILD_STATE_CLONING,
                                          BUILD_STATE_INSTALLING,
                                          BUILD_STATE_BUILDING,
                                          BUILD_STATE_FINISHED)
from readthedocs.builds.models import Build, Version, APIVersion
from readthedocs.builds.signals import build_complete
from readthedocs.builds.syncers import Syncer
from readthedocs.cdn.purge import purge
from readthedocs.core.resolver import resolve_path
from readthedocs.core.symlink import PublicSymlink, PrivateSymlink
from readthedocs.core.utils import send_email, broadcast
from readthedocs.doc_builder.config import load_yaml_config
from readthedocs.doc_builder.constants import DOCKER_LIMITS
from readthedocs.doc_builder.environments import (LocalEnvironment,
                                                  DockerEnvironment)
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.doc_builder.python_environments import Virtualenv, Conda
from readthedocs.projects.models import APIProject
from readthedocs.restapi.client import api as api_v2
from readthedocs.restapi.utils import index_search_request
from readthedocs.search.parse_json import process_all_json_files
from readthedocs.search.utils import process_mkdocs_json
from readthedocs.vcs_support import utils as vcs_support_utils
from readthedocs.worker import app


log = logging.getLogger(__name__)

HTML_ONLY = getattr(settings, 'HTML_ONLY_PROJECTS', ())


class UpdateDocsTask(Task):

    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.
    """

    max_retries = 5
    default_retry_delay = (7 * 60)
    name = __name__ + '.update_docs'

    def __init__(self, build_env=None, python_env=None, config=None,
                 force=False, search=True, localmedia=True,
                 build=None, project=None, version=None):
        self.build_env = build_env
        self.python_env = python_env
        self.build_force = force
        self.build_search = search
        self.build_localmedia = localmedia
        self.build = {}
        if build is not None:
            self.build = build
        self.version = {}
        if version is not None:
            self.version = version
        self.project = {}
        if project is not None:
            self.project = project
        if config is not None:
            self.config = config

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    # pylint: disable=arguments-differ
    def run(self, pk, version_pk=None, build_pk=None, record=True,
            docker=False, search=True, force=False, localmedia=True, **__):
        """
        Run a documentation build.

        This is fully wrapped in exception handling to account for a number of
        failure cases. We first run a few commands in a local build environment,
        but do not report on environment success. This avoids a flicker on the
        build output page where the build is marked as finished in between the
        local environment steps and the docker build steps.

        If a failure is raised, or the build is not successful, return ``False``,
        otherwise, ``True``.

        Unhandled exceptions raise a generic user facing error, which directs
        the user to bug us. It is therefore a benefit to have as few unhandled
        errors as possible.

        :param pk int: Project id
        :param version_pk int: Project Version id
        :param build_pk int: Build id
        :param record bool: record a build object in the database
        :param docker bool: use docker to build the project
        :param search bool: update search
        :param force bool: force Sphinx build
        :param localmedia: update localmedia
        :returns: if build was successful or not
        :rtype: bool
        """
        try:
            self.project = self.get_project(pk)
            self.version = self.get_version(self.project, version_pk)
            self.build = self.get_build(build_pk)
            self.build_search = search
            self.build_localmedia = localmedia
            self.build_force = force
            self.config = None

            setup_successful = self.run_setup(record=record)
            if not setup_successful:
                return False
        # Catch unhandled errors in the setup step
        except Exception as e:  # noqa
            log.exception(
                'An unhandled exception was raised during build setup',
                extra={'tags': {'build': build_pk}}
            )
            self.setup_env.failure = BuildEnvironmentError(
                BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                    build_id=build_pk,
                )
            )
            self.setup_env.update_build(BUILD_STATE_FINISHED)
            return False
        else:
            # No exceptions in the setup step, catch unhandled errors in the
            # build steps
            try:
                self.run_build(record=record, docker=docker)
            except Exception as e:  # noqa
                log.exception(
                    'An unhandled exception was raised during project build',
                    extra={'tags': {'build': build_pk}}
                )
                self.build_env.failure = BuildEnvironmentError(
                    BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                        build_id=build_pk,
                    )
                )
                self.build_env.update_build(BUILD_STATE_FINISHED)
                return False

        return True

    def run_setup(self, record=True):
        """
        Run setup in the local environment.

        Return True if successful.
        """
        self.setup_env = LocalEnvironment(
            project=self.project,
            version=self.version,
            build=self.build,
            record=record,
            update_on_success=False,
        )

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.',
                    status_code=423
                )

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e))
                )

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' % self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
                self.send_notifications()

            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True

    def run_build(self, docker=False, record=True):
        """
        Build the docs in an environment.

        If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
        setting, then build in a Docker environment. Otherwise build locally.
        """
        env_vars = self.get_env_vars()

        if docker or settings.DOCKER_ENABLE:
            env_cls = DockerEnvironment
        else:
            env_cls = LocalEnvironment
        self.build_env = env_cls(project=self.project, version=self.version, config=self.config,
                                 build=self.build, record=record, environment=env_vars)

        # Environment used for building code, usually with Docker
        with self.build_env:

            if self.project.documentation_type == 'auto':
                self.update_documentation_type()

            python_env_cls = Virtualenv
            if self.config.use_conda:
                self._log('Using conda')
                python_env_cls = Conda
            self.python_env = python_env_cls(version=self.version,
                                             build_env=self.build_env,
                                             config=self.config)

            try:
                self.setup_environment()

                # TODO the build object should have an idea of these states, extend
                # the model to include an idea of these outcomes
                outcomes = self.build_docs()
                build_id = self.build.get('id')
            except SoftTimeLimitExceeded:
                raise BuildEnvironmentError(_('Build exited due to time out'))

            # Finalize build and update web servers
            if build_id:
                self.update_app_instances(
                    html=bool(outcomes['html']),
                    search=bool(outcomes['search']),
                    localmedia=bool(outcomes['localmedia']),
                    pdf=bool(outcomes['pdf']),
                    epub=bool(outcomes['epub']),
                )
            else:
                log.warning('No build ID, not syncing files')

        if self.build_env.failed:
            self.send_notifications()

        build_complete.send(sender=Build, build=self.build_env.build)

    @staticmethod
    def get_project(project_pk):
        """Get project from API."""
        project_data = api_v2.project(project_pk).get()
        return APIProject(**project_data)

    @staticmethod
    def get_version(project, version_pk):
        """Ensure we're using a sane version."""
        if version_pk:
            version_data = api_v2.version(version_pk).get()
        else:
            version_data = (api_v2
                            .version(project.slug)
                            .get(slug=LATEST)['objects'][0])
        return APIVersion(**version_data)

    @staticmethod
    def get_build(build_pk):
        """
        Retrieve build object from API.

        :param build_pk: Build primary key
        """
        build = {}
        if build_pk:
            build = api_v2.build(build_pk).get()
        return dict((key, val) for (key, val) in list(build.items())
                    if key not in ['project', 'version', 'resource_uri',
                                   'absolute_uri'])

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        update_imported_docs(self.version.pk)
        commit = self.project.vcs_repo(self.version.slug).commit
        if commit:
            self.build['commit'] = commit

    def get_env_vars(self):
        """Get bash environment variables used for all builder commands."""
        env = {
            'READTHEDOCS': True,
            'READTHEDOCS_VERSION': self.version.slug,
            'READTHEDOCS_PROJECT': self.project.slug
        }

        if self.config.use_conda:
            env.update({
                'CONDA_ENVS_PATH': os.path.join(self.project.doc_path, 'conda'),
                'CONDA_DEFAULT_ENV': self.version.slug,
                'BIN_PATH': os.path.join(self.project.doc_path, 'conda', self.version.slug, 'bin')
            })
        else:
            env.update({
                'BIN_PATH': os.path.join(self.project.doc_path, 'envs', self.version.slug, 'bin')
            })

        return env

    def set_valid_clone(self):
        """Mark on the project that it has been cloned properly."""
        project_data = api_v2.project(self.project.pk).get()
        project_data['has_valid_clone'] = True
        api_v2.project(self.project.pk).put(project_data)
        self.project.has_valid_clone = True

    def update_documentation_type(self):
        """
        Force Sphinx for 'auto' documentation type.

        This used to determine the type and automatically set the documentation
        type to Sphinx for rST and Mkdocs for markdown. It now just forces
        Sphinx, due to markdown support.
        """
        ret = 'sphinx'
        project_data = api_v2.project(self.project.pk).get()
        project_data['documentation_type'] = ret
        api_v2.project(self.project.pk).put(project_data)
        self.project.documentation_type = ret

    def update_app_instances(self, html=False, localmedia=False, search=False,
                             pdf=False, epub=False):
        """
        Update application instances with build artifacts.

        This triggers updates across application instances for html, pdf, epub,
        downloads, and search. Tasks are broadcast to all web servers from here.
        """
        # Update version if we have successfully built HTML output
        try:
            if html:
                version = api_v2.version(self.version.pk)
                version.patch({
                    'active': True,
                    'built': True,
                })
        except HttpClientError:
            log.exception('Updating version failed, skipping file sync: version=%s' % self.version)

        # Broadcast finalization steps to web application instances
        broadcast(
            type='app',
            task=sync_files,
            args=[
                self.project.pk,
                self.version.pk,
            ],
            kwargs=dict(
                hostname=socket.gethostname(),
                html=html,
                localmedia=localmedia,
                search=search,
                pdf=pdf,
                epub=epub,
            ),
            callback=sync_callback.s(version_pk=self.version.pk, commit=self.build['commit']),
        )

    def setup_environment(self):
        """
        Build the virtualenv and install the project into it.

        Always build projects with a virtualenv.

        :param build_env: Build environment to pass commands and execution through.
        """
        self.build_env.update_build(state=BUILD_STATE_INSTALLING)

        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):

            # Check if the python version/build image in the current venv is the
            # same to be used in this build and if it differs, wipe the venv to
            # avoid conflicts.
            if self.python_env.is_obsolete:
                self.python_env.delete_existing_venv_dir()
            else:
                self.python_env.delete_existing_build_dir()

            self.python_env.setup_base()
            self.python_env.save_environment_json()
            self.python_env.install_core_requirements()
            self.python_env.install_user_requirements()
            self.python_env.install_package()

    def build_docs(self):
        """
        Wrapper to all build functions.

        Executes the necessary builds for this task and returns whether the
        build was successful or not.

        :returns: Build outcomes with keys for html, search, localmedia, pdf,
                  and epub
        :rtype: dict
        """
        self.build_env.update_build(state=BUILD_STATE_BUILDING)
        before_build.send(sender=self.version)

        outcomes = defaultdict(lambda: False)
        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
            outcomes['html'] = self.build_docs_html()
            outcomes['search'] = self.build_docs_search()
            outcomes['localmedia'] = self.build_docs_localmedia()
            outcomes['pdf'] = self.build_docs_pdf()
            outcomes['epub'] = self.build_docs_epub()

        after_build.send(sender=self.version)
        return outcomes

    def build_docs_html(self):
        """Build HTML docs."""
        html_builder = get_builder_class(self.project.documentation_type)(
            build_env=self.build_env,
            python_env=self.python_env,
        )
        if self.build_force:
            html_builder.force()
        html_builder.append_conf()
        success = html_builder.build()
        if success:
            html_builder.move()

        # Gracefully attempt to move files via task on web workers.
        try:
            broadcast(type='app', task=move_files,
                      args=[self.version.pk, socket.gethostname()],
                      kwargs=dict(html=True)
                      )
        except socket.error:
            log.exception('move_files task has failed on socket error.')

        return success

    def build_docs_search(self):
        """Build search data with separate build."""
        if self.build_search and self.project.is_type_sphinx:
            return self.build_docs_class('sphinx_search')
        return False

    def build_docs_localmedia(self):
        """Get local media files with separate build."""
        if 'htmlzip' not in self.config.formats:
            return False

        if self.build_localmedia:
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_singlehtmllocalmedia')
        return False

    def build_docs_pdf(self):
        """Build PDF docs."""
        if ('pdf' not in self.config.formats or
            self.project.slug in HTML_ONLY or
{'fraction_non_alphanumeric': 0.07387018307515043, 'fraction_numerical': 0.0012290359749071822, 'mean_word_length': 1.9841075794621026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2859744', 'n_tokens_mistral': 10380, 'n_tokens_neox': 9968, 'n_words': 2839}