<commit_before>import glob
import os
import platform
import subprocess
import sys

from setuptools import Command, Extension, setup
from setuptools.command.test import test as TestCommand


def define_extensions():

    compile_args = ['-fopenmp',
                    '-ffast-math']

    # There are problems with illegal ASM instructions
    # when using the Anaconda distribution (at least on OSX).
    # This could be because Anaconda uses its own assembler?
    # To work around this we do not add -march=native if we
    # know we're dealing with Anaconda
    if 'anaconda' not in sys.version.lower():
        compile_args.append('-march=native')

    return [Extension("glove.glove_cython", ["glove/glove_cython.c"],
                      extra_link_args=["-fopenmp"],
                      extra_compile_args=compile_args),
            Extension("glove.metrics.accuracy_cython",
                      ["glove/metrics/accuracy_cython.c"],
                      extra_link_args=["-fopenmp"],
                      extra_compile_args=compile_args),
            Extension("glove.corpus_cython", ["glove/corpus_cython.cpp"],
                      language='C++',
                      libraries=["stdc++"],
                      extra_link_args=['-std=c++11'] + compile_args,
                      extra_compile_args=['-std=c++11'] + compile_args)]


def set_gcc():
    """
    Try to find and use GCC on OSX for OpenMP support.
    """

    # For macports and homebrew
    patterns = ['/opt/local/bin/gcc-mp-[0-9].[0-9]',
                '/opt/local/bin/gcc-mp-[0-9]',
                '/usr/local/bin/gcc-[0-9].[0-9]',
                '/usr/local/bin/gcc-[0-9]']

    if 'darwin' in platform.platform().lower():

        gcc_binaries = []
        for pattern in patterns:
            gcc_binaries += glob.glob(pattern)
        gcc_binaries.sort()<commit_msg>Make sure c and cpp files are produced.<commit_after>import glob
import os
import platform
import subprocess
import sys

from setuptools import Command, Extension, setup
from setuptools.command.test import test as TestCommand


def define_extensions():

    compile_args = ['-fopenmp',
                    '-ffast-math']

    # There are problems with illegal ASM instructions
    # when using the Anaconda distribution (at least on OSX).
    # This could be because Anaconda uses its own assembler?
    # To work around this we do not add -march=native if we
    # know we're dealing with Anaconda
    if 'anaconda' not in sys.version.lower():
        compile_args.append('-march=native')

    return [Extension("glove.glove_cython", ["glove/glove_cython.pyx"],
                      extra_link_args=["-fopenmp"],
                      extra_compile_args=compile_args),
            Extension("glove.metrics.accuracy_cython",
                      ["glove/metrics/accuracy_cython.pyx"],
                      extra_link_args=["-fopenmp"],
                      extra_compile_args=compile_args),
            Extension("glove.corpus_cython", ["glove/corpus_cython.pyx"],
                      language='C++',
                      libraries=["stdc++"],
                      extra_link_args=['-std=c++11'] + compile_args,
                      extra_compile_args=['-std=c++11'] + compile_args)]


def set_gcc():
    """
    Try to find and use GCC on OSX for OpenMP support.
    """

    # For macports and homebrew
    patterns = ['/opt/local/bin/gcc-mp-[0-9].[0-9]',
                '/opt/local/bin/gcc-mp-[0-9]',
                '/usr/local/bin/gcc-[0-9].[0-9]',
                '/usr/local/bin/gcc-[0-9]']

    if 'darwin' in platform.platform().lower():

        gcc_binaries = []
        for pattern in patterns:
            gcc_binaries += glob.glob(pattern)
        gcc_binaries.sort()
{'fraction_non_alphanumeric': 0.12340085287846482, 'fraction_numerical': 0.008528784648187633, 'mean_word_length': 2.0990916597853015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3030044', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1095, 'n_words': 256}