<commit_before>import subprocess
import sys
import os
import re
import contextlib

# check if pip is installed. If not, raise an ImportError
PIP_INSTALLED = True

try:
    import pip
except ImportError:
    PIP_INSTALLED = False

if not PIP_INSTALLED:
    raise ImportError('pip is not installed.')

def install_and_import(package):
    import importlib
    try:
        importlib.import_module(package)
    except ImportError:
        pip.main(['install', package])
    finally:
        globals()[package] = importlib.import_module(package)

# check if setuptools is installed. If not, install setuptools
# automatically using pip.
install_and_import('setuptools')

from setuptools.command.build_ext import build_ext as _build_ext
from distutils import ccompiler, msvccompiler
from distutils.sysconfig import get_python_inc


## fix compiler and build options
COMPILE_OPTIONS = {
        'msvc': ['/0x', '/EHsc'],
        'mingw32': ['-O3', '-ffast-math', '-march=native'],
        'other': ['-O3', '-ffast-math', '-march=native']
}

LINK_OPTIONS = {
        'msvc': [],
        'mingw32': [],
        'other':[]
}


USE_OPENMP_DEFAULT='1'

# The user can update to use openmp using USE_OPENMP environ variable
if os.environ.get('USE_OPENMP', USE_OPENMP_DEFAULT) == '1':
    if sys.platform == 'win32':
        pass
    else:
        COMPILE_OPTIONS['other'].append('-fopenmp')
        LINK_OPTIONS['other'].append('-fopenmp')<commit_msg>Update setup.py to remove openmp flag<commit_after>import subprocess
import sys
import os
import re
import contextlib

# check if pip is installed. If not, raise an ImportError
PIP_INSTALLED = True

try:
    import pip
except ImportError:
    PIP_INSTALLED = False

if not PIP_INSTALLED:
    raise ImportError('pip is not installed.')

def install_and_import(package):
    import importlib
    try:
        importlib.import_module(package)
    except ImportError:
        pip.main(['install', package])
    finally:
        globals()[package] = importlib.import_module(package)

# check if setuptools is installed. If not, install setuptools
# automatically using pip.
install_and_import('setuptools')

from setuptools.command.build_ext import build_ext as _build_ext
from distutils import ccompiler, msvccompiler
from distutils.sysconfig import get_python_inc


## fix compiler and build options
COMPILE_OPTIONS = {
        'msvc': ['/0x', '/EHsc'],
        'mingw32': ['-O3', '-ffast-math', '-march=native'],
        'other': ['-O3', '-ffast-math', '-march=native']
}

LINK_OPTIONS = {
        'msvc': [],
        'mingw32': [],
        'other':[]
}
{'fraction_non_alphanumeric': 0.1166023166023166, 'fraction_numerical': 0.0069498069498069494, 'mean_word_length': 3.6350626118067977, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2025275', 'n_tokens_mistral': 836, 'n_tokens_neox': 821, 'n_words': 225}