<commit_before>#!/usr/bin/env python
import sys
import os
from os import path
from glob import glob
import subprocess


from setuptools import setup
from distutils.core import Extension
import shutil


pwd = os.path.dirname(__file__)
virtual_env = os.environ.get('VIRTUAL_ENV', '')

includes = ['.']
if virtual_env:
    includes += glob(os.path.join(virtual_env, 'include', 'site', '*'))
else:
    pass


libs = []


compile_args = []
link_args = []


exts = [
    Extension('thinc.learner', ['thinc/learner.cpp'],
              language="c++",
              include_dirs=includes,
              extra_compile_args=['-O3'] + compile_args,
              extra_link_args=['-O3'] + link_args),
    Extension('thinc.weights', ['thinc/weights.cpp'],
              language="c++",
              include_dirs=includes,
              extra_compile_args=['-O3'] + compile_args,
              extra_link_args=['-O3'] + link_args),
    Extension("thinc.features", ["thinc/features.cpp"],
              language="c++", include_dirs=includes,
              extra_compile_args=compile_args,
              extra_link_args=link_args),
    Extension("thinc.search", ["thinc/search.cpp"], language="c++"),
    Extension("thinc.cache", ["thinc/cache.cpp"], include_dirs=includes, language="c++"),
    #Extension("tests.c_test_search", ["tests/c_test_search.cpp"], include_dirs=includes, language="c++")
]


if sys.argv[1] == 'clean':
    print >> sys.stderr, "cleaning .c, .c++ and .so files matching sources"
    map(clean, exts)


setup(
    name='thinc',
    packages=['thinc'],
    version='1.68',
    description="Learn sparse linear models",
    author='<NAME>',
    author_email='<EMAIL>',
    url="http://github.com/syllog1sm/thinc",
    package_data={"thinc": ["*.pyx", "*.pxd", "*.pxi"]},
    ext_modules=exts,
    install_requires=["murmurhash", "cymem", "preshed"],
    setup_requires=["headers_workaround"]
)


import headers_workaround
import sys


include_dir = path.join(sys.prefix, 'include', 'site')
if not path.exists(include_dir):
    os.mkdir(include_dir)
headers_workaround.install_headers(include_dir, 'murmurhash')<commit_msg>* Upd setup for improved header-handling in murmurhash<commit_after>#!/usr/bin/env python
import sys
import os
from os import path
from glob import glob
import subprocess


from setuptools import setup
from distutils.core import Extension
import shutil


pwd = os.path.dirname(__file__)

includes = ['.', path.join(sys.prefix, 'includes')]


libs = []


compile_args = []
link_args = []


exts = [
    Extension('thinc.learner', ['thinc/learner.cpp'],
              language="c++",
              include_dirs=includes,
              extra_compile_args=['-O3'] + compile_args,
              extra_link_args=['-O3'] + link_args),
    Extension('thinc.weights', ['thinc/weights.cpp'],
              language="c++",
              include_dirs=includes,
              extra_compile_args=['-O3'] + compile_args,
              extra_link_args=['-O3'] + link_args),
    Extension("thinc.features", ["thinc/features.cpp"],
              language="c++", include_dirs=includes,
              extra_compile_args=compile_args,
              extra_link_args=link_args),
    Extension("thinc.search", ["thinc/search.cpp"], language="c++"),
    Extension("thinc.cache", ["thinc/cache.cpp"], include_dirs=includes, language="c++"),
    #Extension("tests.c_test_search", ["tests/c_test_search.cpp"], include_dirs=includes, language="c++")
]


if sys.argv[1] == 'clean':
    print >> sys.stderr, "cleaning .c, .c++ and .so files matching sources"
    map(clean, exts)


setup(
    name='thinc',
    packages=['thinc'],
    version='1.68',
    description="Learn sparse linear models",
    author='<NAME>',
    author_email='<EMAIL>',
    url="http://github.com/syllog1sm/thinc",
    package_data={"thinc": ["*.pyx", "*.pxd", "*.pxi"]},
    ext_modules=exts,
    install_requires=["murmurhash", "cymem", "preshed"],
    setup_requires=["headers_workaround"]
)


import headers_workaround
import sys


headers_workaround.install_headers('murmurhash')
{'fraction_non_alphanumeric': 0.1647783251231527, 'fraction_numerical': 0.004433497536945813, 'mean_word_length': 3.557800224466891, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '151200', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1318, 'n_words': 239}