<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
from glob import glob
import os
from setuptools import setup, Extension
import cpuinfo
import numpy as np
from Cython.Build import cythonize


# setup
blosc_sources = []
extra_compile_args = []
include_dirs = []
define_macros = []

# generic setup
blosc_sources += [f for f in glob('c-blosc/blosc/*.c')
                  if 'avx2' not in f and 'sse2' not in f]
blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c')
blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc')
blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c')
include_dirs += [os.path.join('c-blosc', 'blosc')]
include_dirs += glob('c-blosc/internal-complibs/*')
define_macros += [('HAVE_LZ4', 1), ('HAVE_SNAPPY', 1), ('HAVE_ZLIB', 1)]
include_dirs.append(np.get_include())

# determine CPU support for SSE2 and AVX2
cpu_info = cpuinfo.get_cpu_info()

# SSE2
if 'sse2' in cpu_info['flags']:
    print('SSE2 support detected')
    extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED')
    extra_compile_args.append('-msse2')
    blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f]

# AVX2
if 'avx2' in cpu_info['flags']:
    print('AVX2 support detected')
    extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED')<commit_msg>minor<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
from glob import glob
import os
from setuptools import setup, Extension
import cpuinfo
import numpy as np
from Cython.Build import cythonize


# setup
blosc_sources = []
extra_compile_args = []
include_dirs = []
define_macros = []

# generic setup
blosc_sources += [f for f in glob('c-blosc/blosc/*.c')
                  if 'avx2' not in f and 'sse2' not in f]
blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c')
blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc')
blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c')
include_dirs += [os.path.join('c-blosc', 'blosc')]
include_dirs += glob('c-blosc/internal-complibs/*')
define_macros += [('HAVE_LZ4', 1), ('HAVE_SNAPPY', 1), ('HAVE_ZLIB', 1)]
include_dirs.append(np.get_include())

# determine CPU support for SSE2 and AVX2
cpu_info = cpuinfo.get_cpu_info()

# SSE2
if 'sse2' in cpu_info['flags']:
    print('SSE2 detected')
    extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED')
    extra_compile_args.append('-msse2')
    blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f]

# AVX2
if 'avx2' in cpu_info['flags']:
    print('AVX2 detected')
    extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED')
{'fraction_non_alphanumeric': 0.15413819286256644, 'fraction_numerical': 0.015186028853454821, 'mean_word_length': 4.765864332603939, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6280682', 'n_tokens_mistral': 1026, 'n_tokens_neox': 1010, 'n_words': 217}