<commit_before># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

import os
import pkg_resources
import sys
import tempfile
from pkg_resources import (
  Distribution,
  DistributionNotFound,
  EggMetadata)

from twitter.common.lang import Compatibility
from twitter.common.dirutil import chmod_plus_x
from twitter.common.dirutil.chroot import Chroot
from twitter.common.python.importer import EggZipImporter
from twitter.common.python.interpreter import PythonIdentity
from twitter.common.python.marshaller import CodeMarshaller
from twitter.common.python.pex_info import PexInfo
from twitter.common.python.pex import PEX
from twitter.common.python.util import DistributionHelper

BOOTSTRAP_ENVIRONMENT = b"""
import os
import sys

__entry_point__ = None
if '__file__' in locals() and __file__ is not None:
  __entry_point__ = os.path.dirname(__file__)
elif '__loader__' in locals():
  from zipimport import zipimporter
  from pkgutil import ImpLoader
  #if isinstance(__loader__, (builtin_zipimport.zipimporter, EggZipImporter)):
  if hasattr(__loader__, 'archive'):
    __entry_point__ = __loader__.archive
  elif isinstance(__loader__, ImpLoader):
    __entry_point__ = os.path.dirname(__loader__.get_filename())

if __entry_point__ is None:
  sys.stderr.write('Could not launch python executable!\\n')
  sys.exit(2)

sys.path.insert(0, os.path.join(__entry_point__, '.bootstrap'))

from twitter.common.python.importer import monkeypatch
monkeypatch()
del monkeypatch

from twitter.common.python.pex import PEX
PEX(__entry_point__).execute()
"""


class PEXBuilder(object):
  class InvalidDependency(Exception): pass
  class InvalidExecutableSpecification(Exception): pass

  DEPENDENCY_DIR = ".deps"
  BOOTSTRAP_DIR = ".bootstrap"<commit_msg>Pythonpath bugfix in pants.<commit_after># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

import os
import pkg_resources
import sys
import tempfile
from pkg_resources import (
  Distribution,
  DistributionNotFound,
  EggMetadata)

from twitter.common.lang import Compatibility
from twitter.common.dirutil import chmod_plus_x
from twitter.common.dirutil.chroot import Chroot
from twitter.common.python.importer import EggZipImporter
from twitter.common.python.interpreter import PythonIdentity
from twitter.common.python.marshaller import CodeMarshaller
from twitter.common.python.pex_info import PexInfo
from twitter.common.python.pex import PEX
from twitter.common.python.util import DistributionHelper

BOOTSTRAP_ENVIRONMENT = b"""
import os
import sys

__entry_point__ = None
if '__file__' in locals() and __file__ is not None:
  __entry_point__ = os.path.dirname(__file__)
elif '__loader__' in locals():
  from zipimport import zipimporter
  from pkgutil import ImpLoader
  #if isinstance(__loader__, (builtin_zipimport.zipimporter, EggZipImporter)):
  if hasattr(__loader__, 'archive'):
    __entry_point__ = __loader__.archive
  elif isinstance(__loader__, ImpLoader):
    __entry_point__ = os.path.dirname(__loader__.get_filename())

if __entry_point__ is None:
  sys.stderr.write('Could not launch python executable!\\n')
  sys.exit(2)

sys.path[0] = os.path.abspath(sys.path[0])
sys.path.insert(0, os.path.abspath(os.path.join(__entry_point__, '.bootstrap')))

from twitter.common.python.importer import monkeypatch
monkeypatch()
del monkeypatch

from twitter.common.python.pex import PEX
PEX(__entry_point__).execute()
"""


class PEXBuilder(object):
  class InvalidDependency(Exception): pass
  class InvalidExecutableSpecification(Exception): pass

  DEPENDENCY_DIR = ".deps"
  BOOTSTRAP_DIR = ".bootstrap"
{'fraction_non_alphanumeric': 0.1841305998481397, 'fraction_numerical': 0.004176157934700076, 'mean_word_length': 4.677801724137931, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3825812', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1346, 'n_words': 444}