<commit_before>import os
from distutils.core import setup, Extension
import subprocess


def shell_stdout(cmd):
    return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].strip()

for line in shell_stdout('bash -c "source env.sh; env"').split('\n'):
    key, _, value = line.partition("=")
    os.environ[key] = value

include_dirs = os.environ['INC'].replace('-I', '').split()
library_dirs = os.environ['LDIR'].replace('-L', '').split()
libraries = os.environ['LIBS'].replace('-l', '').split()
extra_compile_args = os.environ['CFLAGS'].split()
sources = [
    'loader/src/'+src for src in os.environ['SRCS'].split()
]

module = Extension(
    'aeon_lib',
    sources=sources,
    include_dirs=include_dirs,
    libraries=libraries,
    library_dirs=library_dirs,
    extra_compile_args=extra_compile_args,
)

setup(<commit_msg>Parallelize aeon build: cuts 78% off install time<commit_after>import os
from distutils.core import setup, Extension
import subprocess


def shell_stdout(cmd):
    return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0].strip()

for line in shell_stdout('bash -c "source env.sh; env"').split('\n'):
    key, _, value = line.partition("=")
    os.environ[key] = value

include_dirs = os.environ['INC'].replace('-I', '').split()
library_dirs = os.environ['LDIR'].replace('-L', '').split()
libraries = os.environ['LIBS'].replace('-l', '').split()
extra_compile_args = os.environ['CFLAGS'].split()
sources = [
    'loader/src/'+src for src in os.environ['SRCS'].split()
]

# Parallel build from http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
# monkey-patch for parallel compilation
def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None):
    # those lines are copied from distutils.ccompiler.CCompiler directly
    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs)
    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
    # parallel code
    import multiprocessing.pool
    def _single_compile(obj):
        try: src, ext = build[obj]
        except KeyError: return
        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
    # convert to list, imap is evaluated on-demand
    list(multiprocessing.pool.ThreadPool().imap(_single_compile,objects))
    return objects
import distutils.ccompiler
distutils.ccompiler.CCompiler.compile=parallelCCompile

module = Extension(
    'aeon_lib',
    sources=sources,
    include_dirs=include_dirs,
    libraries=libraries,
    library_dirs=library_dirs,
    extra_compile_args=extra_compile_args,
)

setup(
{'fraction_non_alphanumeric': 0.1416184971098266, 'fraction_numerical': 0.004696531791907515, 'mean_word_length': 5.085714285714285, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1099725', 'n_tokens_mistral': 923, 'n_tokens_neox': 917, 'n_words': 186}