<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
'''Python usage, esp. virtualenv.
'''

import os

from mozharness.base.errors import PythonErrorList
from mozharness.base.log import WARNING, FATAL

# Virtualenv {{{1
virtualenv_config_options = [[
 ["--venv-path", "--virtualenv-path"],
 {"action": "store",
  "dest": "virtualenv_path",
  "default": "venv",
  "help": "Specify the path to the virtualenv top level directory"
 }
],
[["--virtualenv"],
 {"action": "store",
  "dest": "virtualenv",
  "help": "Specify the virtualenv executable to use"
  }
]]

class VirtualenvMixin(object):
    '''BaseScript mixin, designed to create and use virtualenvs.

    Config items:
     * virtualenv_path points to the virtualenv location on disk.
     * virtualenv_modules lists the module names.
     * MODULE_url list points to the module URLs (optional)
    Requires virtualenv to be in PATH.
    Depends on OSMixin
    '''
    python_paths = {}

    def query_virtualenv_path(self):
        c = self.config
        dirs = self.query_abs_dirs()
        if 'abs_virtualenv_dir' in dirs:
            return dirs['abs_virtualenv_dir']
        if os.path.isabs(c['virtualenv_path']):
            return c['virtualenv_path']
        return os.path.join(dirs['abs_work_dir'], c['virtualenv_path'])

    def query_python_path(self, binary="python"):
        """Return the path of a binary inside the virtualenv, if
        c['virtualenv_path'] is set; otherwise return the binary name.
        Otherwise return None
        """
        if binary not in self.python_paths:
            bin_dir = 'bin'
            if self._is_windows():
                bin_dir = 'Scripts'
            virtualenv_path = self.query_virtualenv_path()
            if virtualenv_path:
                self.python_paths[binary] = os.path.abspath(os.path.join(virtualenv_path, bin_dir, binary))
            else:
                self.python_paths[binary] = self.query_exe(binary)
        return self.python_paths[binary]

    def query_package(self, package_name, error_level=WARNING):
        """
        Returns a list of all installed packages
        that contain package_name in their name
        """
        pip = self.query_python_path("pip")
        if not pip:
            self.log("query_package: Program pip not in path", level=error_level)
            return []
        output = self.get_output_from_command(pip + " freeze",
                                              silent=True)
        if not output:
            return []
        packages = output.split()
        return [package for package in packages
                if package.lower().find(package_name.lower()) != -1]

    def create_virtualenv(self):
        c = self.config
        dirs = self.query_abs_dirs()
        venv_path = self.query_virtualenv_path()
        self.info("Creating virtualenv %s" % venv_path)
        virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
        if isinstance(virtualenv, str):
            if not os.path.exists(virtualenv) and not self.which(virtualenv):
                self.add_summary("The executable '%s' is not found; not creating virtualenv!" % virtualenv, level=FATAL)
                return -1
            # allow for [python, virtualenv] in config
            virtualenv = [virtualenv]

        # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
        # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
        if c.get('virtualenv_python_dll'):
            # We may someday want to copy a differently-named dll, but
            # let's not think about that right now =\
            dll_name = os.path.basename(c['virtualenv_python_dll'])
            target = self.query_python_path(dll_name)
            scripts_dir = os.path.dirname(target)
            self.mkdir_p(scripts_dir)
            self.copyfile(c['virtualenv_python_dll'], target)
        else:
            self.mkdir_p(dirs['abs_work_dir'])

        # make this list configurable?
        for module in ('distribute', 'pip'):
            if c.get('%s_url' % module):
                self.download_file(c['%s_url' % module],
                                   parent_dir=dirs['abs_work_dir'])

        virtualenv_options = c.get('virtualenv_options',
                                   ['--no-site-packages', '--distribute'])

        self.run_command(virtualenv + virtualenv_options + [venv_path],
                         cwd=dirs['abs_work_dir'],
                         error_list=PythonErrorList,
                         halt_on_failure=True)
        pip = self.query_python_path("pip")
        for module in c.get('virtualenv_modules', []):
            self.info("Installing %s into virtualenv %s" % (module, venv_path))
            self.run_command([pip, "install", c.get("%s_url" % module,
                                                    module)],
                             error_list=PythonErrorList,
                             halt_on_failure=True)
        self.info("Done creating virtualenv %s." % venv_path)



# __main__ {{{1

if __name__ == '__main__':
    '''TODO: unit tests.
    '''
    pass
<commit_msg>Bug 740147 - VirtualenvMixin default python package server option;r=aki<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
'''Python usage, esp. virtualenv.
'''

import os

from mozharness.base.errors import PythonErrorList
from mozharness.base.log import WARNING, FATAL

# Virtualenv {{{1
virtualenv_config_options = [[
 ["--venv-path", "--virtualenv-path"],
 {"action": "store",
  "dest": "virtualenv_path",
  "default": "venv",
  "help": "Specify the path to the virtualenv top level directory"
 }
],
[["--virtualenv"],
 {"action": "store",
  "dest": "virtualenv",
  "help": "Specify the virtualenv executable to use"
  }
],
[["--pypi-url"],
 {"action": "store",
  "dest": "pypi_url",
  "help": "Base URL of Python Package Index (default http://pypi.python.org/simple/)"
  }
]]

class VirtualenvMixin(object):
    '''BaseScript mixin, designed to create and use virtualenvs.

    Config items:
     * virtualenv_path points to the virtualenv location on disk.
     * virtualenv_modules lists the module names.
     * MODULE_url list points to the module URLs (optional)
    Requires virtualenv to be in PATH.
    Depends on OSMixin
    '''
    python_paths = {}

    def query_virtualenv_path(self):
        c = self.config
        dirs = self.query_abs_dirs()
        if 'abs_virtualenv_dir' in dirs:
            return dirs['abs_virtualenv_dir']
        if os.path.isabs(c['virtualenv_path']):
            return c['virtualenv_path']
        return os.path.join(dirs['abs_work_dir'], c['virtualenv_path'])

    def query_python_path(self, binary="python"):
        """Return the path of a binary inside the virtualenv, if
        c['virtualenv_path'] is set; otherwise return the binary name.
        Otherwise return None
        """
        if binary not in self.python_paths:
            bin_dir = 'bin'
            if self._is_windows():
                bin_dir = 'Scripts'
            virtualenv_path = self.query_virtualenv_path()
            if virtualenv_path:
                self.python_paths[binary] = os.path.abspath(os.path.join(virtualenv_path, bin_dir, binary))
            else:
                self.python_paths[binary] = self.query_exe(binary)
        return self.python_paths[binary]

    def query_package(self, package_name, error_level=WARNING):
        """
        Returns a list of all installed packages
        that contain package_name in their name
        """
        pip = self.query_python_path("pip")
        if not pip:
            self.log("query_package: Program pip not in path", level=error_level)
            return []
        output = self.get_output_from_command(pip + " freeze",
                                              silent=True)
        if not output:
            return []
        packages = output.split()
        return [package for package in packages
                if package.lower().find(package_name.lower()) != -1]

    def create_virtualenv(self):
        c = self.config
        dirs = self.query_abs_dirs()
        venv_path = self.query_virtualenv_path()
        self.info("Creating virtualenv %s" % venv_path)
        virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
        if isinstance(virtualenv, str):
            if not os.path.exists(virtualenv) and not self.which(virtualenv):
                self.add_summary("The executable '%s' is not found; not creating virtualenv!" % virtualenv, level=FATAL)
                return -1
            # allow for [python, virtualenv] in config
            virtualenv = [virtualenv]

        # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
        # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
        if c.get('virtualenv_python_dll'):
            # We may someday want to copy a differently-named dll, but
            # let's not think about that right now =\
            dll_name = os.path.basename(c['virtualenv_python_dll'])
            target = self.query_python_path(dll_name)
            scripts_dir = os.path.dirname(target)
            self.mkdir_p(scripts_dir)
            self.copyfile(c['virtualenv_python_dll'], target)
        else:
            self.mkdir_p(dirs['abs_work_dir'])

        # make this list configurable?
        for module in ('distribute', 'pip'):
            if c.get('%s_url' % module):
                self.download_file(c['%s_url' % module],
                                   parent_dir=dirs['abs_work_dir'])

        virtualenv_options = c.get('virtualenv_options',
                                   ['--no-site-packages', '--distribute'])

        self.run_command(virtualenv + virtualenv_options + [venv_path],
                         cwd=dirs['abs_work_dir'],
                         error_list=PythonErrorList,
                         halt_on_failure=True)
        pip = self.query_python_path("pip")
        command = [pip, "install"]
        pypi_url = c.get("pypi_url")
        if pypi_url:
            command += ["--pypi-url", pypi_url]
        for module in c.get('virtualenv_modules', []):
            self.info("Installing %s into virtualenv %s" % (module, venv_path))
            self.run_command(command + [c.get("%s_url" % module, module)],
                             error_list=PythonErrorList,
                             halt_on_failure=True)
        self.info("Done creating virtualenv %s." % venv_path)



# __main__ {{{1

if __name__ == '__main__':
    '''TODO: unit tests.
    '''
    pass

{'fraction_non_alphanumeric': 0.11520319303338171, 'fraction_numerical': 0.0047169811320754715, 'mean_word_length': 2.251253317605426, 'pattern_counts': {'":': 17, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1972051', 'n_tokens_mistral': 3326, 'n_tokens_neox': 3170, 'n_words': 800}