<commit_before>'''
Documentation Builder Environments
'''

import os
import re
import sys
import logging
import subprocess
import traceback
import socket
from datetime import datetime

from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from docker import Client
from docker.utils import create_host_config
from docker.errors import APIError as DockerAPIError, DockerException
from rest_framework.renderers import JSONRenderer

from readthedocs.builds.constants import BUILD_STATE_FINISHED
from readthedocs.builds.models import BuildCommandResultMixin
from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.api.client import api as api_v1
from readthedocs.restapi.client import api as api_v2
from readthedocs.restapi.serializers import BuildCommandSerializer

from .exceptions import (BuildEnvironmentException, BuildEnvironmentError,
                         BuildEnvironmentWarning)
from .constants import (DOCKER_SOCKET, DOCKER_VERSION, DOCKER_IMAGE,
                        DOCKER_LIMITS, DOCKER_TIMEOUT_EXIT_CODE,
                        DOCKER_OOM_EXIT_CODE, SPHINX_TEMPLATE_DIR,
                        MKDOCS_TEMPLATE_DIR)

log = logging.getLogger(__name__)


class BuildCommand(BuildCommandResultMixin):

    '''Wrap command execution for execution in build environments

    This wraps subprocess commands with some logic to handle exceptions,
    logging, and setting up the env for the build command.

    This acts a mapping of sorts to the API reprensentation of the
    :py:cls:`readthedocs.builds.models.BuildCommandResult` model.

    :param command: string or array of command parameters
    :param cwd: current working path for the command
    :param shell: execute command in shell, default=False
    :param environment: environment variables to add to environment
    :type environment: dict
    :param combine_output: combine stdout/stderr, default=True
    :param input_data: data to pass in on stdin
    :type input_data: str
    :param build_env: build environment to use to execute commands
    :param bin_path: binary path to add to PATH resolution
    :param description: a more grokable description of the command being run
    '''

    def __init__(self, command, cwd=None, shell=False, environment=None,
                 combine_output=True, input_data=None, build_env=None,
                 bin_path=None, description=None):
        self.command = command
        self.shell = shell
        if cwd is None:
            cwd = os.getcwd()
        self.cwd = cwd
        self.environment = os.environ.copy()
        if environment is not None:
            assert 'PATH' not in environment, "PATH can't be set"
            self.environment.update(environment)

        self.combine_output = combine_output
        self.input_data = input_data
        self.build_env = build_env
        self.output = None
        self.error = None
        self.start_time = None
        self.end_time = None

        self.bin_path = bin_path
        self.description = ''
        if description is not None:
            self.description = description
        self.exit_code = None

    def __str__(self):
        # TODO do we want to expose the full command here?
        output = u''
        if self.output is not None:
            output = self.output.encode('utf-8')
        return '\n'.join([self.get_command(), output])

    def run(self):
        '''Set up subprocess and execute command

        :param cmd_input: input to pass to command in STDIN
        :type cmd_input: str
        :param combine_output: combine STDERR into STDOUT
        '''
        log.info("Running: '%s' [%s]", self.get_command(), self.cwd)

        self.start_time = datetime.utcnow()
        stdout = subprocess.PIPE
        stderr = subprocess.PIPE
        stdin = None
        if self.input_data is not None:
            stdin = subprocess.PIPE
        if self.combine_output:
            stderr = subprocess.STDOUT

        environment = {}
        environment.update(self.environment)
        environment['READTHEDOCS'] = 'True'
        if self.build_env is not None:
            environment['READTHEDOCS_VERSION'] = self.build_env.version.slug
            environment['READTHEDOCS_PROJECT'] = self.build_env.project.slug
        if 'DJANGO_SETTINGS_MODULE' in environment:
            del environment['DJANGO_SETTINGS_MODULE']
        if 'PYTHONPATH' in environment:
            del environment['PYTHONPATH']
        if self.bin_path is not None:
            env_paths = environment.get('BIN_PATH', '').split(':')
            env_paths.insert(0, self.bin_path)
            environment['PATH'] = ':'.join(env_paths)

        try:
            proc = subprocess.Popen(
                self.command,
                shell=self.shell,
                cwd=self.cwd,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
                env=environment,
            )
            cmd_input = None
            if self.input_data is not None:
                cmd_input = self.input_data

            cmd_output = proc.communicate(input=cmd_input)
            (cmd_stdout, cmd_stderr) = cmd_output
            try:
                self.output = cmd_stdout.decode('utf-8', 'replace')
            except (TypeError, AttributeError):
                self.output = None
            try:
                self.error = cmd_stderr.decode('utf-8', 'replace')
            except (TypeError, AttributeError):
                self.error = None
            self.exit_code = proc.returncode
        except OSError:
            self.error = traceback.format_exc()<commit_msg>Properly use PATH on local python installs<commit_after>'''
Documentation Builder Environments
'''

import os
import re
import sys
import logging
import subprocess
import traceback
import socket
from datetime import datetime

from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from docker import Client
from docker.utils import create_host_config
from docker.errors import APIError as DockerAPIError, DockerException
from rest_framework.renderers import JSONRenderer

from readthedocs.builds.constants import BUILD_STATE_FINISHED
from readthedocs.builds.models import BuildCommandResultMixin
from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.api.client import api as api_v1
from readthedocs.restapi.client import api as api_v2
from readthedocs.restapi.serializers import BuildCommandSerializer

from .exceptions import (BuildEnvironmentException, BuildEnvironmentError,
                         BuildEnvironmentWarning)
from .constants import (DOCKER_SOCKET, DOCKER_VERSION, DOCKER_IMAGE,
                        DOCKER_LIMITS, DOCKER_TIMEOUT_EXIT_CODE,
                        DOCKER_OOM_EXIT_CODE, SPHINX_TEMPLATE_DIR,
                        MKDOCS_TEMPLATE_DIR)

log = logging.getLogger(__name__)


class BuildCommand(BuildCommandResultMixin):

    '''Wrap command execution for execution in build environments

    This wraps subprocess commands with some logic to handle exceptions,
    logging, and setting up the env for the build command.

    This acts a mapping of sorts to the API reprensentation of the
    :py:cls:`readthedocs.builds.models.BuildCommandResult` model.

    :param command: string or array of command parameters
    :param cwd: current working path for the command
    :param shell: execute command in shell, default=False
    :param environment: environment variables to add to environment
    :type environment: dict
    :param combine_output: combine stdout/stderr, default=True
    :param input_data: data to pass in on stdin
    :type input_data: str
    :param build_env: build environment to use to execute commands
    :param bin_path: binary path to add to PATH resolution
    :param description: a more grokable description of the command being run
    '''

    def __init__(self, command, cwd=None, shell=False, environment=None,
                 combine_output=True, input_data=None, build_env=None,
                 bin_path=None, description=None):
        self.command = command
        self.shell = shell
        if cwd is None:
            cwd = os.getcwd()
        self.cwd = cwd
        self.environment = os.environ.copy()
        if environment is not None:
            assert 'PATH' not in environment, "PATH can't be set"
            self.environment.update(environment)

        self.combine_output = combine_output
        self.input_data = input_data
        self.build_env = build_env
        self.output = None
        self.error = None
        self.start_time = None
        self.end_time = None

        self.bin_path = bin_path
        self.description = ''
        if description is not None:
            self.description = description
        self.exit_code = None

    def __str__(self):
        # TODO do we want to expose the full command here?
        output = u''
        if self.output is not None:
            output = self.output.encode('utf-8')
        return '\n'.join([self.get_command(), output])

    def run(self):
        '''Set up subprocess and execute command

        :param cmd_input: input to pass to command in STDIN
        :type cmd_input: str
        :param combine_output: combine STDERR into STDOUT
        '''
        log.info("Running: '%s' [%s]", self.get_command(), self.cwd)

        self.start_time = datetime.utcnow()
        stdout = subprocess.PIPE
        stderr = subprocess.PIPE
        stdin = None
        if self.input_data is not None:
            stdin = subprocess.PIPE
        if self.combine_output:
            stderr = subprocess.STDOUT

        environment = {}
        environment.update(self.environment)
        environment['READTHEDOCS'] = 'True'
        if self.build_env is not None:
            environment['READTHEDOCS_VERSION'] = self.build_env.version.slug
            environment['READTHEDOCS_PROJECT'] = self.build_env.project.slug
        if 'DJANGO_SETTINGS_MODULE' in environment:
            del environment['DJANGO_SETTINGS_MODULE']
        if 'PYTHONPATH' in environment:
            del environment['PYTHONPATH']
        if self.bin_path is not None:
            env_paths = environment.get('PATH', '').split(':')
            env_paths.insert(0, self.bin_path)
            environment['PATH'] = ':'.join(env_paths)

        try:
            proc = subprocess.Popen(
                self.command,
                shell=self.shell,
                cwd=self.cwd,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
                env=environment,
            )
            cmd_input = None
            if self.input_data is not None:
                cmd_input = self.input_data

            cmd_output = proc.communicate(input=cmd_input)
            (cmd_stdout, cmd_stderr) = cmd_output
            try:
                self.output = cmd_stdout.decode('utf-8', 'replace')
            except (TypeError, AttributeError):
                self.output = None
            try:
                self.error = cmd_stderr.decode('utf-8', 'replace')
            except (TypeError, AttributeError):
                self.error = None
            self.exit_code = proc.returncode
        except OSError:
            self.error = traceback.format_exc()
{'fraction_non_alphanumeric': 0.07226786818623554, 'fraction_numerical': 0.0010601643254704478, 'mean_word_length': 2.3902365977837676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3505687', 'n_tokens_mistral': 3125, 'n_tokens_neox': 3033, 'n_words': 895}