<commit_before>#------------------------------------------------------------------------------
# Copyright (c) 2008, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD license
# available at http://www.enthought.com/licenses/BSD.txt and may be
# redistributed only under the conditions described in the aforementioned
# license.
#
# <NAME>
#------------------------------------------------------------------------------

# imports from standard library
import os, sys, glob, re, platform
from os import path
from urlparse import urlsplit, urljoin
from urllib2 import urlopen
from logging import error, warning, info, debug

# imports from 3rd party libraries
from setuptools.package_index import distros_for_url
import pkg_resources

entry_patt = re.compile( "^([\w\-_]+):\ .*" )


def get_platform():
    """ Return finer-grained platform names, corresponding to supported EPD
    versions."""
    
    (PLAT, PLAT_VER) = platform.dist()[0:2]
    
    # Map RedHat to Enthought repo names
    if PLAT.lower().startswith("redhat"):
        PLAT = "redhat"
        if PLAT_VER.startswith("3"):
            PLAT_VER = "3"
        elif PLAT_VER.startswith("4"):
            PLAT_VER = "4"
        elif PLAT_VER.startswith("5"):
            PLAT_VER = "5"

    # Ubuntu returns debian...check /etc/issue too
    elif PLAT.lower().startswith("debian"):
        if path.exists("/etc/issue"):
            fh = open("/etc/issue", "r")
            lines = fh.readlines()
            fh.close()
            patt = re.compile("^([\w]+) ([\w\.]+).*")
            for line in lines:
                match = patt.match(line)
                if match is not None:
                    plat = match.group(1).lower()
                    if plat == "ubuntu":
                        PLAT = plat
                        PLAT_VER = match.group(2).lower()
                break

    # Windows
    elif sys.platform.lower().startswith("win"):
        PLAT = "windows"
        # this returns "xp" for Windows XP and "vista" for Windows Vista
        # XXX If we want to support Windows 2k we need to check this
        PLAT_VER = platform.release().lower()

    # setuptools get_platform() finds the right info for OSX
    elif sys.platform.lower().startswith("darwin"):
        (PLAT, PLAT_VER) = pkg_resources.get_platform().split( "-" )[0:2]
    
    return (PLAT, PLAT_VER)


def remove_eggs_from_path(search_path, fix_names=False):
    """
    Returns a copy of search_path with all eggs (directories or zip files)
    removed.  Eggs are identified by the ".egg" extension in the name.
    If fix_names is True, the dir names in the path are made absolute.
    Note: files with a .zip extension are removed as well.
    """
    new_path = []
    for name in search_path:
        if fix_names:
            name = os.path.normpath(path.abspath(name))
        if not (os.path.splitext(name)[-1].lower() in [".egg", ".zip"]
                or glob.glob(os.path.join(name, "*.egg-info"))):
            new_path.append(name)<commit_msg>Ensure that the system site-packages will always be included after stripping off certain sys.path entries to find repos.<commit_after>#------------------------------------------------------------------------------
# Copyright (c) 2008, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD license
# available at http://www.enthought.com/licenses/BSD.txt and may be
# redistributed only under the conditions described in the aforementioned
# license.
#
# <NAME>
#------------------------------------------------------------------------------

# imports from standard library
from distutils import sysconfig
import os, sys, glob, re, platform
from os import path
from urlparse import urlsplit, urljoin
from urllib2 import urlopen
from logging import error, warning, info, debug

# imports from 3rd party libraries
from setuptools.package_index import distros_for_url
import pkg_resources

entry_patt = re.compile( "^([\w\-_]+):\ .*" )


def get_platform():
    """ Return finer-grained platform names, corresponding to supported EPD
    versions."""
    
    (PLAT, PLAT_VER) = platform.dist()[0:2]
    
    # Map RedHat to Enthought repo names
    if PLAT.lower().startswith("redhat"):
        PLAT = "redhat"
        if PLAT_VER.startswith("3"):
            PLAT_VER = "3"
        elif PLAT_VER.startswith("4"):
            PLAT_VER = "4"
        elif PLAT_VER.startswith("5"):
            PLAT_VER = "5"

    # Ubuntu returns debian...check /etc/issue too
    elif PLAT.lower().startswith("debian"):
        if path.exists("/etc/issue"):
            fh = open("/etc/issue", "r")
            lines = fh.readlines()
            fh.close()
            patt = re.compile("^([\w]+) ([\w\.]+).*")
            for line in lines:
                match = patt.match(line)
                if match is not None:
                    plat = match.group(1).lower()
                    if plat == "ubuntu":
                        PLAT = plat
                        PLAT_VER = match.group(2).lower()
                break

    # Windows
    elif sys.platform.lower().startswith("win"):
        PLAT = "windows"
        # this returns "xp" for Windows XP and "vista" for Windows Vista
        # XXX If we want to support Windows 2k we need to check this
        PLAT_VER = platform.release().lower()

    # setuptools get_platform() finds the right info for OSX
    elif sys.platform.lower().startswith("darwin"):
        (PLAT, PLAT_VER) = pkg_resources.get_platform().split( "-" )[0:2]
    
    return (PLAT, PLAT_VER)


def remove_eggs_from_path(search_path, fix_names=False):
    """
    Returns a copy of search_path with all eggs (directories or zip files)
    removed.  Eggs are identified by the ".egg" extension in the name.
    If fix_names is True, the dir names in the path are made absolute.
    Note: files with a .zip extension are removed as well.
    """
    new_path = []
    for name in search_path:
        if fix_names:
            name = os.path.normpath(path.abspath(name))
        if not (os.path.splitext(name)[-1].lower() in [".egg", ".zip"]
                or glob.glob(os.path.join(name, "*.egg-info"))):
            new_path.append(name)
            
    # If for some reason the main site-packages has .egg-info files in it, be sure to add it
    # to our path, because otherwise it would not be added.
    site_packages = sysconfig.get_python_lib()
    if site_packages not in new_path:
        new_path.append(site_packages)
    
{'fraction_non_alphanumeric': 0.15516717325227963, 'fraction_numerical': 0.0060790273556231, 'mean_word_length': 2.0341171046565236, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5275782', 'n_tokens_mistral': 1953, 'n_tokens_neox': 1883, 'n_words': 603}