<commit_before># -*- coding: utf-8 -*-
"""An abstraction over virtualenv and Conda environments."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import json
import logging
import os
import shutil
import six
from builtins import object, open

from django.conf import settings

from readthedocs.doc_builder.config import ConfigWrapper
from readthedocs.doc_builder.constants import DOCKER_IMAGE
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.projects.models import Feature

log = logging.getLogger(__name__)


class PythonEnvironment(object):

    """An isolated environment into which Python packages can be installed."""

    def __init__(self, version, build_env, config=None):
        self.version = version
        self.project = version.project
        self.build_env = build_env
        if config:
            self.config = config
        else:
            self.config = ConfigWrapper(version=version, yaml_config={})
        # Compute here, since it's used a lot
        self.checkout_path = self.project.checkout_path(self.version.slug)

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    def delete_existing_build_dir(self):
        # Handle deleting old build dir
        build_dir = os.path.join(
            self.venv_path(),
            'build')
        if os.path.exists(build_dir):
            self._log('Removing existing build directory')
            shutil.rmtree(build_dir)

    def delete_existing_venv_dir(self):
        venv_dir = self.venv_path()
        # Handle deleting old venv dir
        if os.path.exists(venv_dir):
            self._log('Removing existing venv directory')
            shutil.rmtree(venv_dir)

    def install_package(self):
        if self.config.install_project:
            if self.config.pip_install or getattr(settings, 'USE_PIP_INSTALL', False):
                extra_req_param = ''
                if self.config.extra_requirements:
                    extra_req_param = '[{0}]'.format(
                        ','.join(self.config.extra_requirements))
                self.build_env.run(
                    'python',
                    self.venv_bin(filename='pip'),
                    'install',
                    '--ignore-installed',
                    '--cache-dir',
                    self.project.pip_cache_path,
                    '.{0}'.format(extra_req_param),
                    cwd=self.checkout_path,
                    bin_path=self.venv_bin()
                )
            else:
                self.build_env.run(
                    'python',
                    'setup.py',
                    'install',
                    '--force',
                    cwd=self.checkout_path,
                    bin_path=self.venv_bin()
                )

    def venv_bin(self, filename=None):
        """
        Return path to the virtualenv bin path, or a specific binary.

        :param filename: If specified, add this filename to the path return
        :returns: Path to virtualenv bin or filename in virtualenv bin
        """
        parts = [self.venv_path(), 'bin']
        if filename is not None:
            parts.append(filename)
        return os.path.join(*parts)

    def environment_json_path(self):
        """Return the path to the ``readthedocs-environment.json`` file."""
        return os.path.join(
            self.venv_path(),
            'readthedocs-environment.json',
        )

    @property
    def is_obsolete(self):
        """
        Determine if the environment is obsolete for different reasons.

        It checks the the data stored at ``readthedocs-environment.json`` and
        compares it with the one to be used. In particular:

        * the Python version (e.g. 2.7, 3, 3.6, etc)
        * the Docker image name
        * the Docker image hash

        :returns: ``True`` when it's obsolete and ``False`` otherwise

        :rtype: bool
        """
        # Always returns False if we don't have information about what Python
        # version/Docker image was used to create the venv as backward
        # compatibility.
        if not os.path.exists(self.environment_json_path()):
            return False

        try:
            with open(self.environment_json_path(), 'r') as fpath:
                environment_conf = json.load(fpath)
        except (IOError, TypeError, KeyError, ValueError):
            log.warning('Unable to read/parse readthedocs-environment.json file')
            # We remove the JSON file here to avoid cycling over time with a
            # corrupted file.
            os.remove(self.environment_json_path())
            return True

        env_python = environment_conf.get('python', {})
        env_build = environment_conf.get('build', {})

        # By defaulting non-existent options to ``None`` we force a wipe since
        # we don't know how the environment was created
        env_python_version = env_python.get('version', None)
        env_build_image = env_build.get('image', None)
        env_build_hash = env_build.get('hash', None)

        build_image = self.config.build_image or DOCKER_IMAGE
        # If the user define the Python version just as a major version
        # (e.g. ``2`` or ``3``) we won't know exactly which exact version was
        # used to create the venv but we can still compare it against the new
        # one coming from the project version config.
        return any([
            env_python_version != self.config.python_full_version,
            env_build_image != build_image,
            env_build_hash != self.build_env.image_hash,
        ])

    def save_environment_json(self):
        """Save on disk Python and build image versions used to create the venv."""
        build_image = self.config.build_image or DOCKER_IMAGE
        data = {
            'python': {
                'version': self.config.python_full_version,
            },
            'build': {
                'image': build_image,
                'hash': self.build_env.image_hash,
            },
        }

        with open(self.environment_json_path(), 'w') as fpath:
            # Compatibility for Py2 and Py3. ``io.TextIOWrapper`` expects
            # unicode but ``json.dumps`` returns str in Py2.
            fpath.write(six.text_type(json.dumps(data)))


class Virtualenv(PythonEnvironment):<commit_msg>Save Docker image data in JSON file only for DockerBuildEnvironment (#3897)<commit_after># -*- coding: utf-8 -*-
"""An abstraction over virtualenv and Conda environments."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import json
import logging
import os
import shutil
from builtins import object, open

import six
from django.conf import settings

from readthedocs.doc_builder.config import ConfigWrapper
from readthedocs.doc_builder.constants import DOCKER_IMAGE
from readthedocs.doc_builder.environments import DockerBuildEnvironment
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.projects.models import Feature

log = logging.getLogger(__name__)


class PythonEnvironment(object):

    """An isolated environment into which Python packages can be installed."""

    def __init__(self, version, build_env, config=None):
        self.version = version
        self.project = version.project
        self.build_env = build_env
        if config:
            self.config = config
        else:
            self.config = ConfigWrapper(version=version, yaml_config={})
        # Compute here, since it's used a lot
        self.checkout_path = self.project.checkout_path(self.version.slug)

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    def delete_existing_build_dir(self):
        # Handle deleting old build dir
        build_dir = os.path.join(
            self.venv_path(),
            'build')
        if os.path.exists(build_dir):
            self._log('Removing existing build directory')
            shutil.rmtree(build_dir)

    def delete_existing_venv_dir(self):
        venv_dir = self.venv_path()
        # Handle deleting old venv dir
        if os.path.exists(venv_dir):
            self._log('Removing existing venv directory')
            shutil.rmtree(venv_dir)

    def install_package(self):
        if self.config.install_project:
            if self.config.pip_install or getattr(settings, 'USE_PIP_INSTALL', False):
                extra_req_param = ''
                if self.config.extra_requirements:
                    extra_req_param = '[{0}]'.format(
                        ','.join(self.config.extra_requirements))
                self.build_env.run(
                    'python',
                    self.venv_bin(filename='pip'),
                    'install',
                    '--ignore-installed',
                    '--cache-dir',
                    self.project.pip_cache_path,
                    '.{0}'.format(extra_req_param),
                    cwd=self.checkout_path,
                    bin_path=self.venv_bin()
                )
            else:
                self.build_env.run(
                    'python',
                    'setup.py',
                    'install',
                    '--force',
                    cwd=self.checkout_path,
                    bin_path=self.venv_bin()
                )

    def venv_bin(self, filename=None):
        """
        Return path to the virtualenv bin path, or a specific binary.

        :param filename: If specified, add this filename to the path return
        :returns: Path to virtualenv bin or filename in virtualenv bin
        """
        parts = [self.venv_path(), 'bin']
        if filename is not None:
            parts.append(filename)
        return os.path.join(*parts)

    def environment_json_path(self):
        """Return the path to the ``readthedocs-environment.json`` file."""
        return os.path.join(
            self.venv_path(),
            'readthedocs-environment.json',
        )

    @property
    def is_obsolete(self):
        """
        Determine if the environment is obsolete for different reasons.

        It checks the the data stored at ``readthedocs-environment.json`` and
        compares it with the one to be used. In particular:

        * the Python version (e.g. 2.7, 3, 3.6, etc)
        * the Docker image name
        * the Docker image hash

        :returns: ``True`` when it's obsolete and ``False`` otherwise

        :rtype: bool
        """
        # Always returns False if we don't have information about what Python
        # version/Docker image was used to create the venv as backward
        # compatibility.
        if not os.path.exists(self.environment_json_path()):
            return False

        try:
            with open(self.environment_json_path(), 'r') as fpath:
                environment_conf = json.load(fpath)
        except (IOError, TypeError, KeyError, ValueError):
            log.warning('Unable to read/parse readthedocs-environment.json file')
            # We remove the JSON file here to avoid cycling over time with a
            # corrupted file.
            os.remove(self.environment_json_path())
            return True

        env_python = environment_conf.get('python', {})
        env_build = environment_conf.get('build', {})

        # By defaulting non-existent options to ``None`` we force a wipe since
        # we don't know how the environment was created
        env_python_version = env_python.get('version', None)
        env_build_image = env_build.get('image', None)
        env_build_hash = env_build.get('hash', None)

        if isinstance(self.build_env, DockerBuildEnvironment):
            build_image = self.config.build_image or DOCKER_IMAGE
            image_hash = self.build_env.image_hash
        else:
            # e.g. LocalBuildEnvironment
            build_image = None
            image_hash = None

        # If the user define the Python version just as a major version
        # (e.g. ``2`` or ``3``) we won't know exactly which exact version was
        # used to create the venv but we can still compare it against the new
        # one coming from the project version config.
        return any([
            env_python_version != self.config.python_full_version,
            env_build_image != build_image,
            env_build_hash != image_hash,
        ])

    def save_environment_json(self):
        """Save on disk Python and build image versions used to create the venv."""
        data = {
            'python': {
                'version': self.config.python_full_version,
            },
        }

        if isinstance(self.build_env, DockerBuildEnvironment):
            build_image = self.config.build_image or DOCKER_IMAGE
            data.update({
                'build': {
                    'image': build_image,
                    'hash': self.build_env.image_hash,
                },
            })

        with open(self.environment_json_path(), 'w') as fpath:
            # Compatibility for Py2 and Py3. ``io.TextIOWrapper`` expects
            # unicode but ``json.dumps`` returns str in Py2.
            fpath.write(six.text_type(json.dumps(data)))


class Virtualenv(PythonEnvironment):
{'fraction_non_alphanumeric': 0.08922670191672175, 'fraction_numerical': 0.002203128442388191, 'mean_word_length': 1.9909949483856797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3009500', 'n_tokens_mistral': 3756, 'n_tokens_neox': 3579, 'n_words': 1011}