<commit_before># Setup script for megaman: scalable manifold learning
# LICENSE: Simplified BSD https://github.com/mmp2/megaman/blob/master/LICENSE

import io
import os
import re
import sys
import subprocess
import importlib.machinery


def read(path, encoding='utf-8'):
    path = os.path.join(os.path.dirname(__file__), path)
    with io.open(path, encoding=encoding) as fp:
        return fp.read()


def version(path):
    """Obtain the packge version from a python file e.g. pkg/__init__.py

    See <https://packaging.python.org/en/latest/single_source_version.html>.
    """
    version_file = read(path)<commit_msg>Update setup.py<commit_after># Setup script for megaman: scalable manifold learning
# LICENSE: Simplified BSD https://github.com/mmp2/megaman/blob/master/LICENSE

import io
import os
import re
import sys
import subprocess

PY2 = sys.version_info[0] == 2
PY3 = not PY2
if PY3:
    import importlib.machinery


def read(path, encoding='utf-8'):
    path = os.path.join(os.path.dirname(__file__), path)
    with io.open(path, encoding=encoding) as fp:
        return fp.read()


def version(path):
    """Obtain the packge version from a python file e.g. pkg/__init__.py

    See <https://packaging.python.org/en/latest/single_source_version.html>.
    """
    version_file = read(path)
{'fraction_non_alphanumeric': 0.11715160796324656, 'fraction_numerical': 0.007656967840735069, 'mean_word_length': 3.9135338345864663, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7668685', 'n_tokens_mistral': 435, 'n_tokens_neox': 422, 'n_words': 114}