<commit_before>import os
import shutil
from distutils import log
from distutils.command.build_ext import build_ext
from distutils.core import Extension
from distutils.dist import Distribution
from distutils.sysconfig import get_config_vars

try:
    from Cython.Build import cythonize
except ImportError:
    cythonize = None

if os.path.dirname(__file__):
    os.chdir(os.path.dirname(__file__))
log.set_verbosity(3)

opt, = get_config_vars('OPT')
os.environ['OPT'] = ' '.join(flag for flag in opt.split() if flag != '-Wstrict-prototypes')
extra_compile_args = ['-march=native', '-O3']

sources = ['ptdebug.cpp', 'ptdebug_x86.cpp', 'ptdebug_x64.cpp',
           'ptdebug_x86_on_x64.cpp', 'ptdebug_x32.cpp', 'ptdebug_arm.cpp', 'ptproc.cpp']

if cythonize is None:
    if os.path.exists('_cptbox.cpp'):
        sources.insert(0, '_cptbox.cpp')
    else:
        print 'You need to have either cython or _cptbox.cpp prebuilt.'
        raise SystemExit(1)
else:
    sources.insert(0, '_cptbox.pyx')

ext_modules = [Extension('_cptbox', sources=sources,
                         language='c++', libraries=['rt'],
                         extra_compile_args=extra_compile_args)]

if cythonize is not None:
    ext_modules = cythonize(ext_modules)

distribution = Distribution({
    'ext_modules': ext_modules
})

command = build_ext(distribution)
command.finalize_options()
command.run()

shutil.copy2(command.get_ext_fullpath('_cptbox'),
             command.get_ext_filename('_cptbox'))
<commit_msg>ARM doesn't like -march=native #128<commit_after>import os
import shutil
from distutils import log
from distutils.command.build_ext import build_ext
from distutils.core import Extension
from distutils.dist import Distribution
from distutils.sysconfig import get_config_vars

try:
    from Cython.Build import cythonize
except ImportError:
    cythonize = None

if os.path.dirname(__file__):
    os.chdir(os.path.dirname(__file__))
log.set_verbosity(3)

opt, = get_config_vars('OPT')
os.environ['OPT'] = ' '.join(flag for flag in opt.split() if flag != '-Wstrict-prototypes')

if os.uname()[4].startswith('arm'):
    extra_compile_args = ['-O3']
else:
    extra_compile_args = ['-march=native', '-O3']

sources = ['ptdebug.cpp', 'ptdebug_x86.cpp', 'ptdebug_x64.cpp',
           'ptdebug_x86_on_x64.cpp', 'ptdebug_x32.cpp', 'ptdebug_arm.cpp', 'ptproc.cpp']

if cythonize is None:
    if os.path.exists('_cptbox.cpp'):
        sources.insert(0, '_cptbox.cpp')
    else:
        print 'You need to have either cython or _cptbox.cpp prebuilt.'
        raise SystemExit(1)
else:
    sources.insert(0, '_cptbox.pyx')

ext_modules = [Extension('_cptbox', sources=sources,
                         language='c++', libraries=['rt'],
                         extra_compile_args=extra_compile_args)]

if cythonize is not None:
    ext_modules = cythonize(ext_modules)

distribution = Distribution({
    'ext_modules': ext_modules
})

command = build_ext(distribution)
command.finalize_options()
command.run()

shutil.copy2(command.get_ext_fullpath('_cptbox'),
             command.get_ext_filename('_cptbox'))

{'fraction_non_alphanumeric': 0.12132582864290181, 'fraction_numerical': 0.011569731081926203, 'mean_word_length': 3.4185082872928176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4295346', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1064, 'n_words': 221}