<commit_before>"""
twitter.common.python support for interpreter environments.
"""
from __future__ import absolute_import

try:
  from numbers import Integral
except ImportError:
  Integral = (int, long)

from collections import defaultdict
import os
import re
import subprocess
import sys

from .base import maybe_requirement, maybe_requirement_list
from .tracer import Tracer

from pkg_resources import (
    find_distributions,
    Distribution,
    Requirement,
)

TRACER = Tracer(predicate=Tracer.env_filter('PEX_VERBOSE'),
    prefix='twitter.common.python.interpreter: ')


# Determine in the most platform-compatible way possible the identity of the interpreter
# and its known packages.
ID_PY = b"""
import sys

if hasattr(sys, 'pypy_version_info'):
  subversion = 'PyPy'
elif sys.platform.startswith('java'):
  subversion = 'Jython'
else:
  subversion = 'CPython'

print("%s %s %s %s" % (
  subversion,
  sys.version_info[0],
  sys.version_info[1],
  sys.version_info[2]))

setuptools_path = None
try:
  import pkg_resources
except ImportError:
  sys.exit(0)

requirements = {}
for item in sys.path:
  for dist in pkg_resources.find_distributions(item):
    requirements[str(dist.as_requirement())] = dist.location

for requirement_str, location in requirements.items():
  rs = requirement_str.split('==', 2)
  if len(rs) == 2:
    print('%s %s %s' % (rs[0], rs[1], location))
"""


class PythonCapability(list):
  def __init__(self, requirements=None):
    super(PythonCapability, self).__init__(maybe_requirement_list(requirements or []))


class PythonIdentity(object):
  class Error(Exception): pass
  class InvalidError(Error): pass
  class UnknownRequirement(Error): pass

  # TODO(wickman)  Support interpreter-specific versions, e.g. PyPy-2.2.1
  HASHBANGS = {
    'CPython': 'python%(major)d.%(minor)d',
    'Jython': 'jython',
    'PyPy': 'pypy',
  }

  @classmethod
  def get_subversion(cls):
    if hasattr(sys, 'pypy_version_info'):
      subversion = 'PyPy'
    elif sys.platform.startswith('java'):
      subversion = 'Jython'
    else:
      subversion = 'CPython'
    return subversion

  @classmethod
  def get(cls):
    return cls(cls.get_subversion(), sys.version_info[0], sys.version_info[1], sys.version_info[2])

  @classmethod
  def from_id_string(cls, id_string):
    values = id_string.split()
    if len(values) != 4:
      raise cls.InvalidError("Invalid id string: %s" % id_string)
    return cls(str(values[0]), int(values[1]), int(values[2]), int(values[3]))

  @classmethod
  def from_path(cls, dirname):
    interp, version = dirname.split('-')
    major, minor, patch = version.split('.')
    return cls(str(interp), int(major), int(minor), int(patch))

  def __init__(self, interpreter, major, minor, patch):
    for var in (major, minor, patch):
      assert isinstance(var, Integral)
    self._interpreter = interpreter
    self._version = (major, minor, patch)

  @property
  def interpreter(self):
    return self._interpreter

  @property
  def version(self):
    return self._version

  @property
  def requirement(self):
    return self.distribution.as_requirement()

  @property
  def distribution(self):
    return Distribution(project_name=self._interpreter, version='.'.join(map(str, self._version)))

  @classmethod
  def parse_requirement(cls, requirement, default_interpreter='CPython'):
    if isinstance(requirement, Requirement):
      return requirement
    elif isinstance(requirement, str):
      try:
        requirement = Requirement.parse(requirement)
      except ValueError:
        try:
          requirement = Requirement.parse('%s%s' % (default_interpreter, requirement))<commit_msg>Support unicode literal requirement strings in python <3.<commit_after>"""
twitter.common.python support for interpreter environments.
"""
from __future__ import absolute_import

try:
  from numbers import Integral
except ImportError:
  Integral = (int, long)

from collections import defaultdict
import os
import re
import subprocess
import sys

from .base import maybe_requirement, maybe_requirement_list
from .compatibility import string
from .tracer import Tracer

from pkg_resources import (
    find_distributions,
    Distribution,
    Requirement,
)

TRACER = Tracer(predicate=Tracer.env_filter('PEX_VERBOSE'),
    prefix='twitter.common.python.interpreter: ')


# Determine in the most platform-compatible way possible the identity of the interpreter
# and its known packages.
ID_PY = b"""
import sys

if hasattr(sys, 'pypy_version_info'):
  subversion = 'PyPy'
elif sys.platform.startswith('java'):
  subversion = 'Jython'
else:
  subversion = 'CPython'

print("%s %s %s %s" % (
  subversion,
  sys.version_info[0],
  sys.version_info[1],
  sys.version_info[2]))

setuptools_path = None
try:
  import pkg_resources
except ImportError:
  sys.exit(0)

requirements = {}
for item in sys.path:
  for dist in pkg_resources.find_distributions(item):
    requirements[str(dist.as_requirement())] = dist.location

for requirement_str, location in requirements.items():
  rs = requirement_str.split('==', 2)
  if len(rs) == 2:
    print('%s %s %s' % (rs[0], rs[1], location))
"""


class PythonCapability(list):
  def __init__(self, requirements=None):
    super(PythonCapability, self).__init__(maybe_requirement_list(requirements or []))


class PythonIdentity(object):
  class Error(Exception): pass
  class InvalidError(Error): pass
  class UnknownRequirement(Error): pass

  # TODO(wickman)  Support interpreter-specific versions, e.g. PyPy-2.2.1
  HASHBANGS = {
    'CPython': 'python%(major)d.%(minor)d',
    'Jython': 'jython',
    'PyPy': 'pypy',
  }

  @classmethod
  def get_subversion(cls):
    if hasattr(sys, 'pypy_version_info'):
      subversion = 'PyPy'
    elif sys.platform.startswith('java'):
      subversion = 'Jython'
    else:
      subversion = 'CPython'
    return subversion

  @classmethod
  def get(cls):
    return cls(cls.get_subversion(), sys.version_info[0], sys.version_info[1], sys.version_info[2])

  @classmethod
  def from_id_string(cls, id_string):
    values = id_string.split()
    if len(values) != 4:
      raise cls.InvalidError("Invalid id string: %s" % id_string)
    return cls(str(values[0]), int(values[1]), int(values[2]), int(values[3]))

  @classmethod
  def from_path(cls, dirname):
    interp, version = dirname.split('-')
    major, minor, patch = version.split('.')
    return cls(str(interp), int(major), int(minor), int(patch))

  def __init__(self, interpreter, major, minor, patch):
    for var in (major, minor, patch):
      assert isinstance(var, Integral)
    self._interpreter = interpreter
    self._version = (major, minor, patch)

  @property
  def interpreter(self):
    return self._interpreter

  @property
  def version(self):
    return self._version

  @property
  def requirement(self):
    return self.distribution.as_requirement()

  @property
  def distribution(self):
    return Distribution(project_name=self._interpreter, version='.'.join(map(str, self._version)))

  @classmethod
  def parse_requirement(cls, requirement, default_interpreter='CPython'):
    if isinstance(requirement, Requirement):
      return requirement
    elif isinstance(requirement, string):
      try:
        requirement = Requirement.parse(requirement)
      except ValueError:
        try:
          requirement = Requirement.parse('%s%s' % (default_interpreter, requirement))
{'fraction_non_alphanumeric': 0.12214256729338563, 'fraction_numerical': 0.0052752603814419045, 'mean_word_length': 4.373546511627907, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4662011', 'n_tokens_mistral': 2397, 'n_tokens_neox': 2351, 'n_words': 580}