<commit_before>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import contextlib
import glob
import logging
import os
import shutil
import subprocess
import sys
import tempfile

from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL
from util import read_deps, merge_deps, print_deps, platform_tag

LOGGER = logging.getLogger(__name__)

# /path/to/infra
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*'


class NoWheelException(Exception):
  def __init__(self, name, version, build, source_sha):
    super(NoWheelException, self).__init__(
        'No matching wheel found for (%s==%s (build %s_%s))' %
        (name, version, build, source_sha))


def check_pydistutils():
  if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')):
    print >> sys.stderr, '\n'.join([
      '',
      '',
      '=========== ERROR ===========',
      'You have a ~/.pydistutils.cfg file, which interferes with the ',
      'infra virtualenv environment. Please move it to the side and bootstrap ',
      'again. Once infra has bootstrapped, you may move it back.',
      '',
      'Upstream bug: https://github.com/pypa/virtualenv/issues/88/',
      ''
    ])
    sys.exit(1)


def ls(prefix):
  from pip._vendor import requests  # pylint: disable=E0611
  data = requests.get(STORAGE_URL, params=dict(
      prefix=prefix,
      fields='items(name,md5Hash)'
  )).json()
  entries = data.get('items', [])
  for entry in entries:
    entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex')
    entry['local'] = False
  # Also look in the local cache
  entries.extend([
    {'name': fname, 'md5Hash': None, 'local': True}
    for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH,
                                        prefix.split('/')[-1] + '*'))])
  return entries


def sha_for(deps_entry):
  if 'rev' in deps_entry:
    return deps_entry['rev']
  else:
    return deps_entry['gs'].split('.')[0]


def get_links(deps):
  import pip.wheel  # pylint: disable=E0611
  plat_tag = platform_tag()

  links = []

  for name, dep in deps.iteritems():
    version, source_sha = dep['version'] , sha_for(dep)
    prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
                                         source_sha)
    generic_link = None
    binary_link = None
    local_link = None

    for entry in ls(prefix):
      fname = entry['name'].split('/')[-1]
      md5hash = entry['md5Hash']
      wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
      if not wheel_info:
        LOGGER.warn('Skipping invalid wheel: %r', fname)
        continue

      if pip.wheel.Wheel(fname).supported():
        if entry['local']:
          link = LOCAL_OBJECT_URL.format(entry['name'])
          local_link = link
          continue
        else:
          link = OBJECT_URL.format(entry['name'], md5hash)
        if fname.endswith('none-any.whl'):
          if generic_link:
            LOGGER.error(
              'Found more than one generic matching wheel for %r: %r',
              prefix, dep)
            continue
          generic_link = link
        elif plat_tag in fname:
          if binary_link:
            LOGGER.error(
              'Found more than one binary matching wheel for %r: %r',
              prefix, dep)
            continue
          binary_link = link

    if not binary_link and not generic_link and not local_link:
      raise NoWheelException(name, version, dep['build'], source_sha)

    links.append(local_link or binary_link or generic_link)

  return links


@contextlib.contextmanager
def html_index(links):
  tf = tempfile.mktemp('.html')
  try:
    with open(tf, 'w') as f:
      print >> f, '<html><body>'
      for link in links:
        print >> f, '<a href="%s">wat</a>' % link
      print >> f, '</body></html>'
    yield tf
  finally:
    os.unlink(tf)


def install(deps):
  bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
  pip = os.path.join(sys.prefix, bin_dir, 'pip')

  links = get_links(deps)
  with html_index(links) as ipath:
    requirements = []
    # TODO(iannucci): Do this as a requirements.txt
    for name, deps_entry in deps.iteritems():
      if not deps_entry.get('implicit'):
        requirements.append('%s==%s' % (name, deps_entry['version']))
    subprocess.check_call(
        [pip, 'install', '--no-index', '--download-cache',
         os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements)


def activate_env(env, deps, quiet):
  if hasattr(sys, 'real_prefix'):
    LOGGER.error('Already activated environment!')
    return<commit_msg>Added default value for the quiet parameter<commit_after>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import contextlib
import glob
import logging
import os
import shutil
import subprocess
import sys
import tempfile

from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL
from util import read_deps, merge_deps, print_deps, platform_tag

LOGGER = logging.getLogger(__name__)

# /path/to/infra
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*'


class NoWheelException(Exception):
  def __init__(self, name, version, build, source_sha):
    super(NoWheelException, self).__init__(
        'No matching wheel found for (%s==%s (build %s_%s))' %
        (name, version, build, source_sha))


def check_pydistutils():
  if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')):
    print >> sys.stderr, '\n'.join([
      '',
      '',
      '=========== ERROR ===========',
      'You have a ~/.pydistutils.cfg file, which interferes with the ',
      'infra virtualenv environment. Please move it to the side and bootstrap ',
      'again. Once infra has bootstrapped, you may move it back.',
      '',
      'Upstream bug: https://github.com/pypa/virtualenv/issues/88/',
      ''
    ])
    sys.exit(1)


def ls(prefix):
  from pip._vendor import requests  # pylint: disable=E0611
  data = requests.get(STORAGE_URL, params=dict(
      prefix=prefix,
      fields='items(name,md5Hash)'
  )).json()
  entries = data.get('items', [])
  for entry in entries:
    entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex')
    entry['local'] = False
  # Also look in the local cache
  entries.extend([
    {'name': fname, 'md5Hash': None, 'local': True}
    for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH,
                                        prefix.split('/')[-1] + '*'))])
  return entries


def sha_for(deps_entry):
  if 'rev' in deps_entry:
    return deps_entry['rev']
  else:
    return deps_entry['gs'].split('.')[0]


def get_links(deps):
  import pip.wheel  # pylint: disable=E0611
  plat_tag = platform_tag()

  links = []

  for name, dep in deps.iteritems():
    version, source_sha = dep['version'] , sha_for(dep)
    prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
                                         source_sha)
    generic_link = None
    binary_link = None
    local_link = None

    for entry in ls(prefix):
      fname = entry['name'].split('/')[-1]
      md5hash = entry['md5Hash']
      wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
      if not wheel_info:
        LOGGER.warn('Skipping invalid wheel: %r', fname)
        continue

      if pip.wheel.Wheel(fname).supported():
        if entry['local']:
          link = LOCAL_OBJECT_URL.format(entry['name'])
          local_link = link
          continue
        else:
          link = OBJECT_URL.format(entry['name'], md5hash)
        if fname.endswith('none-any.whl'):
          if generic_link:
            LOGGER.error(
              'Found more than one generic matching wheel for %r: %r',
              prefix, dep)
            continue
          generic_link = link
        elif plat_tag in fname:
          if binary_link:
            LOGGER.error(
              'Found more than one binary matching wheel for %r: %r',
              prefix, dep)
            continue
          binary_link = link

    if not binary_link and not generic_link and not local_link:
      raise NoWheelException(name, version, dep['build'], source_sha)

    links.append(local_link or binary_link or generic_link)

  return links


@contextlib.contextmanager
def html_index(links):
  tf = tempfile.mktemp('.html')
  try:
    with open(tf, 'w') as f:
      print >> f, '<html><body>'
      for link in links:
        print >> f, '<a href="%s">wat</a>' % link
      print >> f, '</body></html>'
    yield tf
  finally:
    os.unlink(tf)


def install(deps):
  bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
  pip = os.path.join(sys.prefix, bin_dir, 'pip')

  links = get_links(deps)
  with html_index(links) as ipath:
    requirements = []
    # TODO(iannucci): Do this as a requirements.txt
    for name, deps_entry in deps.iteritems():
      if not deps_entry.get('implicit'):
        requirements.append('%s==%s' % (name, deps_entry['version']))
    subprocess.check_call(
        [pip, 'install', '--no-index', '--download-cache',
         os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements)


def activate_env(env, deps, quiet=False):
  if hasattr(sys, 'real_prefix'):
    LOGGER.error('Already activated environment!')
    return
{'fraction_non_alphanumeric': 0.13808532778355878, 'fraction_numerical': 0.005827263267429761, 'mean_word_length': 2.8245125348189415, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512308', 'n_tokens_mistral': 3153, 'n_tokens_neox': 3080, 'n_words': 801}