<commit_before>"""
To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
created. Clearly there is huge need to use conforming syntax. But many changes
(e.g. ``property``, ``hasattr`` in ``2.5``) can be rewritten in pure python.
"""
import sys
import imp
import os
try:
    import importlib
except:
    pass

is_py3k = sys.hexversion >= 0x03000000
is_py33 = sys.hexversion >= 0x03030000


def find_module_py33(string, path=None):
    mod_info = (None, None, None)
    loader = None
    if path is not None:
        # Check for the module in the specidied path
        loader = importlib.machinery.PathFinder.find_module(string, path)
    else:
        # Check for the module in sys.path
        loader = importlib.machinery.PathFinder.find_module(string, sys.path)
        if loader is None:
            # Fallback to find builtins
            loader = importlib.find_loader(string)

    if loader is None:
        raise ImportError

    try:
        if (loader.is_package(string)):
            mod_info = (None, os.path.dirname(loader.path), True)
        else:
            filename = loader.get_filename(string)
            if filename and os.path.exists(filename):
                mod_info = (open(filename, 'U'), filename, False)
            else:
                mod_info = (None, filename, False)
    except AttributeError:
        mod_info = (None, loader.load_module(string).__name__, False)

    return mod_info


def find_module_pre_py33(string, path=None):
    mod_info = None
    if path is None:
        mod_info = imp.find_module(string)
    else:
        mod_info = imp.find_module(string, path)

    return (mod_info[0], mod_info[1], mod_info[2][2] == imp.PKG_DIRECTORY)<commit_msg>Clean up find_module_pre_py33<commit_after>"""
To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
created. Clearly there is huge need to use conforming syntax. But many changes
(e.g. ``property``, ``hasattr`` in ``2.5``) can be rewritten in pure python.
"""
import sys
import imp
import os
try:
    import importlib
except:
    pass

is_py3k = sys.hexversion >= 0x03000000
is_py33 = sys.hexversion >= 0x03030000


def find_module_py33(string, path=None):
    mod_info = (None, None, None)
    loader = None
    if path is not None:
        # Check for the module in the specidied path
        loader = importlib.machinery.PathFinder.find_module(string, path)
    else:
        # Check for the module in sys.path
        loader = importlib.machinery.PathFinder.find_module(string, sys.path)
        if loader is None:
            # Fallback to find builtins
            loader = importlib.find_loader(string)

    if loader is None:
        raise ImportError

    try:
        if (loader.is_package(string)):
            mod_info = (None, os.path.dirname(loader.path), True)
        else:
            filename = loader.get_filename(string)
            if filename and os.path.exists(filename):
                mod_info = (open(filename, 'U'), filename, False)
            else:
                mod_info = (None, filename, False)
    except AttributeError:
        mod_info = (None, loader.load_module(string).__name__, False)

    return mod_info


def find_module_pre_py33(string, path=None):
    module_file, module_path, description = imp.find_module(string, path)
    module_type = description[2]
    return module_file, module_path, module_type is imp.PKG_DIRECTORY
{'fraction_non_alphanumeric': 0.09859154929577464, 'fraction_numerical': 0.020246478873239437, 'mean_word_length': 2.75027502750275, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189617', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1035, 'n_words': 309}