<commit_before>import os
import re
import sys
import distutils.sysconfig
import distutils.util

ARCH_BUILD_FLAGS = {
    'universal': {
        'target': '10.5',
        'cflags': '-isysroot /Developer/SDKs/MacOSX10.5.sdk -arch i386 -arch ppc -arch ppc64 -arch x86_64',
        'cc': 'gcc-4.2',
    },
    'ppc64': {
        'target': '10.5',
        'cflags': '-isysroot /Developer/SDKs/MacOSX10.5.sdk -arch x86_64',
        'cc': 'gcc-4.2',
    },
    'x86_64': {
        'target': '10.5',
        'cflags': '-isysroot / -arch x86_64',
        'cc': 'gcc-4.2',
    },
    'fat3': {
        'target': '10.5',
        'cflags': '-isysroot / -arch i386 -arch ppc -arch x86_64',
        'cc': 'gcc-4.2',
    },
    'intel': {
        'target': '10.5',
        'cflags': '-isysroot / -arch i386 -arch x86_64',
        'cc': 'gcc-4.2',
    },
    'i386': {
        'target': '10.3',
        'cflags': '-isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch i386',
        'cc': 'gcc-4.0',
    },
    'ppc': {
        'target': '10.3',
        'cflags': '-isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch ppc',
        'cc': 'gcc-4.0',
    },
    'fat': {
        'target': '10.3',
        'cflags': '-isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch i386 -arch ppc',
        'cc': 'gcc-4.0',
    },
}

def main_executable_path():
    basepath = os.path.dirname(__file__)
    return os.path.join(basepath, 'prebuilt', 'main')

def main():
    basepath = os.path.dirname(__file__)
    dest = main_executable_path()
    src = os.path.join(basepath, 'src', 'main.m')
    if os.path.exists(dest) and (os.stat(dest).st_mtime < os.stat(src).st_mtime):
        return dest

    builddir = os.path.join(basepath, 'prebuilt')
    if not os.path.exists(builddir):
        os.makedirs(builddir)
    cfg = distutils.sysconfig.get_config_vars()

    BASE_CFLAGS = cfg['CFLAGS']
    BASE_CFLAGS = BASE_CFLAGS.replace('-dynamic', '')
    BASE_CFLAGS += ' -bundle -framework Foundation -framework AppKit'
    while True:
        x = re.sub('-arch\s+\S+', '', BASE_CFLAGS)
        if x == BASE_CFLAGS:
            break
        BASE_CFLAGS=x

    while True:
        x = re.sub('-isysroot\s+\S+', '', BASE_CFLAGS)
        if x == BASE_CFLAGS:
            break
        BASE_CFLAGS=x

    arch = distutils.util.get_platform().split('-')[-1]
    if sys.prefix.startswith('/System') and \
            sys.version_info[:2] == (2,5):
        arch = "fat"

    arch_flags = ARCH_BUILD_FLAGS[arch]
    CC=arch_flags['cc']
    CFLAGS = BASE_CFLAGS + ' ' + arch_flags['cflags']
    os.environ['MACOSX_DEPLOYMENT_TARGET'] = arch_flags['target']
    os.system('"%(CC)s" -o "%(dest)s" "%(src)s" %(CFLAGS)s' % locals())
    return dest


if __name__ == '__main__':
    main()<commit_msg>Use distutils.ccompiler to compile main.m<commit_after>import os
import re
import distutils.sysconfig
import distutils.util
import distutils.ccompiler

def main_executable_path():
    basepath = os.path.dirname(__file__)
    return os.path.join(basepath, 'prebuilt', 'main')

def main():
    basepath = os.path.dirname(__file__)
    dest = main_executable_path()
    src = os.path.join(basepath, 'src', 'main.m')
    if os.path.exists(dest) and (os.stat(dest).st_mtime < os.stat(src).st_mtime):
        return dest

    builddir = os.path.join(basepath, 'prebuilt')
    if not os.path.exists(builddir):
        os.makedirs(builddir)
    cfg = distutils.sysconfig.get_config_vars()
    os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg['MACOSX_DEPLOYMENT_TARGET']
    CFLAGS = cfg['CFLAGS']
    architectures = re.findall(r'-arch\s+(\S+)', CFLAGS)
    isysroot = re.findall(r'-isysroot\s+(\S+)', CFLAGS)[0]
    compiler = distutils.ccompiler.new_compiler(verbose=True)
    extra_args = []
    for arch in architectures:
        extra_args += ['-arch', arch]
    extra_args += ['-isysroot', isysroot]
    compiler.compile([src], extra_postargs=extra_args)
    obj = compiler.object_filenames([src])[0]
    # add link args to extra args
    extra_args += ['-bundle', '-framework', 'Foundation', '-framework', 'AppKit']
    compiler.link_executable([obj], dest, extra_postargs=extra_args)
    os.remove(obj)
    return dest


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.17234803337306318, 'fraction_numerical': 0.025744934445768773, 'mean_word_length': 2.7099911582670204, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '648854', 'n_tokens_mistral': 1644, 'n_tokens_neox': 1561, 'n_words': 291}