<commit_before>
import os, sys, subprocess
import argparse, platform
import time


class CytherError(Exception):
    """A helpful custom error to be called when a general python error just doesn't make sense in context"""
    def __init__(self, *args, **kwargs):
        super(CytherError, self).__init__(*args, **kwargs)


try:
    from shutil import which
except ImportError:
    raise CytherError("The current version of Python doesn't support the function 'which', normally located in shutil")


CYTHER_FAIL = 0
CYTHER_SUCCESS = 1
ERROR_PASSOFF = -3
INTERVAL = .25

CYTHONIZABLE_FILE_EXTS = ('.pyx', '.py')


NOT_NEEDED_MESSAGE = "Module '{}' does not have to be included, or has no .get_include() method"

LIB_A_MISSING_MESSAGE = '''
Currently, cyther does not support the automatic construction of 'libpython{0}.a',
so you will have to do it yourself. There are several tools that can help you with
this, including dlltool, gendef, pexport, reimp. All of which are a part of mingw32.

The general idea is to first look into the python{0}.dll and figure out all the
objects that belong to that file. Then, take the names of these objects, and put
them in a .def file. Take this file, and pass it to dll tool to tell it what to
extract from the python{0}.dll and inject into the libpython{0}.a static library.
'''


ASSUMPTIONS = """
Assumptions cyther makes about your system:
1) Cython and gcc are both installed, and accessible from the system console
2) Python supports 'shutil.which'
3) Your environment path variable is able to be found by `shutil.which`
4) gcc can work with the option -l pythonXY (libpythonXY.a exists in your python libs directory)
5) Almost any gcc compiled C program will work on Windows
6) Python's 'libs' and 'include' directories are in the same drive that python is installed
"""

PYTHON_DIRECTORY = sys.exec_prefix
DRIVE_AND_NAME = os.path.splitdrive(PYTHON_DIRECTORY)
PYTHON_NAME = os.path.basename(DRIVE_AND_NAME[1]).lower()
DRIVE = DRIVE_AND_NAME[0]
if not DRIVE:
    DRIVE = os.path.normpath('/')


def where(cmd, mode=os.X_OK, path=None, error=True, crawl=False, datafile=False):
    """This function will wrap the shutil.which function to return the abspath every time, or a empty string"""
    found = {}
    error_string = ''
    if crawl:
        if isinstance(cmd, str):
            cmd = [cmd]
        source = DRIVE
        for root, dirs, files in os.walk(source):
            for container in (dirs, files):
                for w in container:
                    if w in cmd:
                        string = os.path.abspath(os.path.join(source, root, w))
                        if 'python' in string.lower():
                            error_string += "'python' is in '{}'\n".format(string)
                            if w in found:
                                if len(found[w]) > len(string):
                                    found[w] = string
                                else:
                                    error_string += 'not setting it\n'
                            else:
                                found[w] = string
        for item in cmd:
            if item not in found:
                raise CytherError("The item '{}' was not found searching drive '{}'\n\n{}".format(item, DRIVE, error_string))
    else:
        if isinstance(cmd, str):
            cmd = [cmd]
        for find_this in cmd:
            raw_result = which(find_this, mode, path)
            if raw_result:
                found[find_this] = os.path.abspath(raw_result)
            else:
                raise CytherError("Could not find '{}' in the path".format(find_this))
    return found

<commit_msg>Trying to fix travis problems, take 5<commit_after>
import os, sys, subprocess
import argparse, platform
import time


class CytherError(Exception):
    """A helpful custom error to be called when a general python error just doesn't make sense in context"""
    def __init__(self, *args, **kwargs):
        super(CytherError, self).__init__(*args, **kwargs)


try:
    from shutil import which
except ImportError:
    raise CytherError("The current version of Python doesn't support the function 'which', normally located in shutil")


CYTHER_FAIL = 0
CYTHER_SUCCESS = 1
ERROR_PASSOFF = -3
INTERVAL = .25

CYTHONIZABLE_FILE_EXTS = ('.pyx', '.py')


NOT_NEEDED_MESSAGE = "Module '{}' does not have to be included, or has no .get_include() method"

LIB_A_MISSING_MESSAGE = '''
Currently, cyther does not support the automatic construction of 'libpython{0}.a',
so you will have to do it yourself. There are several tools that can help you with
this, including dlltool, gendef, pexport, reimp. All of which are a part of mingw32.

The general idea is to first look into the python{0}.dll and figure out all the
objects that belong to that file. Then, take the names of these objects, and put
them in a .def file. Take this file, and pass it to dll tool to tell it what to
extract from the python{0}.dll and inject into the libpython{0}.a static library.
'''


ASSUMPTIONS = """
Assumptions cyther makes about your system:
1) Cython and gcc are both installed, and accessible from the system console
2) Python supports 'shutil.which'
3) Your environment path variable is able to be found by `shutil.which`
4) gcc can work with the option -l pythonXY (libpythonXY.a exists in your python libs directory)
5) Almost any gcc compiled C program will work on Windows
6) Python's 'libs' and 'include' directories are in the same drive that python is installed
"""

PYTHON_DIRECTORY = sys.exec_prefix
DRIVE_AND_NAME = os.path.splitdrive(PYTHON_DIRECTORY)
PYTHON_NAME = os.path.basename(DRIVE_AND_NAME[1]).lower()
DRIVE = DRIVE_AND_NAME[0]
if not DRIVE:
    DRIVE = os.path.normpath('/')


def where(cmd, mode=os.X_OK, path=None, error=True, crawl=False, datafile=False):
    """This function will wrap the shutil.which function to return the abspath every time, or a empty string"""
    found = {}
    error_string = ''
    if crawl:
        if isinstance(cmd, str):
            cmd = [cmd]
        source = DRIVE
        for root, dirs, files in os.walk(source):
            for container in (dirs, files):
                for w in container:
                    if w in cmd:
                        string = os.path.abspath(os.path.join(source, root, w))
                        if PYTHON_NAME in string.lower():
                            error_string += "'{}' is in '{}'\n".format(PYTHON_NAME, string)
                            if w in found:
                                if len(found[w]) > len(string):
                                    found[w] = string
                                else:
                                    error_string += 'not setting it\n'
                            else:
                                found[w] = string
        for item in cmd:
            if item not in found:
                raise CytherError("The item '{}' was not found searching drive '{}'\n\n{}".format(item, DRIVE, error_string))
    else:
        if isinstance(cmd, str):
            cmd = [cmd]
        for find_this in cmd:
            raw_result = which(find_this, mode, path)
            if raw_result:
                found[find_this] = os.path.abspath(raw_result)
            else:
                raise CytherError("Could not find '{}' in the path".format(find_this))
    return found


{'fraction_non_alphanumeric': 0.08888284860016309, 'fraction_numerical': 0.00530035335689046, 'mean_word_length': 2.4101019462465247, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1627536', 'n_tokens_mistral': 2142, 'n_tokens_neox': 2065, 'n_words': 822}