<commit_before>from __future__ import print_function
from distutils.core import setup, Extension, Command 
from distutils.command.build_ext import build_ext
import os
import subprocess
import sys
import shutil
import glob
from Cython.Build import cythonize

#os.environ['CXX'] = 'g++'
#os.environ['ARCHFLAGS'] ="-arch x86_64"

# add this to fix build on mac osx mavericks
#os.environ['ARCHFLAGS'] ="-Wno-error=unused-command-line-argument-hard-error-in-future"

copy_args = sys.argv[1:]
AAF_ROOT = None
if '--aaf-root' in copy_args:
    index = copy_args.index('--aaf-root')
    AAF_ROOT = copy_args[index+1]
    del copy_args[index]
    del copy_args[index]
else:
    AAF_ROOT = os.environ.get("AAF_ROOT")
    
USE_AAF_SDK_DEBUG = bool(int(os.environ.get("USE_AAF_SDK_DEBUG", "1")))
NTHREADS= int(os.environ.get("NTHREADS",0))

space = '   '
if AAF_ROOT is None:

    print( space, "Unable to locate AAF Development libraries.")
    print( space, "Please specify with --aaf-root or AAF_ROOT env variable")
    print( space, "AAF SDK can be found from http://aaf.sourceforge.net")
    print( space, "Pre-built devel libraries can be found here")
    print( space, "http://sourceforge.net/projects/aaf/files/AAF-devel-libs/1.1.6")
    sys.exit(-1)
    
if not os.path.exists(AAF_ROOT):
    print(space, "AAF_ROOT direcotry does not exist: %s" % AAF_ROOT)
    sys.exit(-1)
    
AAF_INCLUDE = os.path.join(AAF_ROOT,'include')

AAF_LIB = os.path.join(AAF_ROOT,'lib', 'debug')

AAF_COM = os.path.join(AAF_ROOT,'bin', 'debug')

ext_extra = {
    'include_dirs': ['headers',AAF_INCLUDE],
    'library_dirs': [AAF_LIB, AAF_COM],
    'libraries': ['aaflib','aafiid', 'com-api'],
}

if sys.platform.startswith('linux'):
    ext_extra['extra_link_args'] = ['-Wl,-R$ORIGIN']

WIN_ARCH = 'Win32'

if sys.platform.startswith('win'):
    import platform
    if platform.architecture()[0] == '64bit':
        WIN_ARCH = 'x64'
    if '--debug' in copy_args or USE_AAF_SDK_DEBUG:
        ext_extra['library_dirs'] = [os.path.join(AAF_ROOT,WIN_ARCH ,'Debug','Refimpl')]
        ext_extra['libraries'] = ['AAFD', 'AAFIIDD']
    else:
        ext_extra['library_dirs'] = [os.path.join(AAF_ROOT,WIN_ARCH ,'Release','Refimpl')]
        ext_extra['libraries'] = ['AAF', 'AAFIID']

    ext_extra['library_dirs'].extend([os.path.join(AAF_ROOT, 'lib'),
                                        os.path.join(AAF_ROOT, 'bin')])
    
print("AAF_ROOT =",AAF_ROOT)


ext_modules = []
for dirname, dirnames, filenames in os.walk("aaf", topdown=True):
    for filename in filenames:
        if filename.startswith('.') or os.path.splitext(filename)[1] != '.pyx':
            continue

        path = os.path.join(dirname, filename)
        name = "aaf." + os.path.splitext(os.path.basename(path))[0]
        
        sources = [path]

        extra_src_dir = os.path.join(os.path.dirname(__file__), 'aaf', os.path.splitext(filename)[0])
        extra_src = glob.glob(os.path.join(extra_src_dir, '*.cpp'))
        
        sources.extend(extra_src)
        extension = Extension(name,
                              sources=sources,
                              language="c++",
                              **ext_extra)

        ext_modules.append(extension)
    break

def get_com_api(debug=True):
    if sys.platform.startswith("win"):
        dir = os.path.join(AAF_ROOT,'%s' % str(WIN_ARCH))
        if debug or USE_AAF_SDK_DEBUG:
            dir = os.path.join(dir, "Debug")
        else:
            dir = os.path.join(dir, "Release")

        for dirname in ext_extra['library_dirs']:
            com_api = os.path.join(dirname, 'AAFCOAPI.dll')
            libaafintp =  os.path.join(dirname, 'aafext', 'AAFINTP.dll')
            libaafpgapi =  os.path.join(dirname, 'aafext', 'AAFPGAPI.dll')
            #print(os.path.basename(com_api), '=', com_api)
            if all([os.path.exists(item) for item in (com_api,libaafintp,libaafpgapi)]):
                return com_api, libaafintp, libaafpgapi
        raise Exception("Unable to find AAFCOAPI.dll, AAFINTP.dll, AAFPGAPI.dll")
    ext = '.so'
    if sys.platform == 'darwin':
        ext = '.dylib'
    dir = os.path.join(AAF_ROOT, 'bin')
    if debug or USE_AAF_SDK_DEBUG:
        dir = os.path.join(dir, 'debug')
    
    com_api = os.path.join(dir, 'libcom-api' + ext)
    libaafintp = os.path.join(dir, 'aafext', 'libaafintp' + ext)
    libaafpgapi = os.path.join(dir, 'aafext', 'libaafpgapi' + ext)
    
    return com_api, libaafintp, libaafpgapi

def copy_com_api(debug=True):
    com_api, libaafintp, libaafpgapi = get_com_api(debug)
    
    for item in [com_api, libaafintp, libaafpgapi]:
        print(os.path.basename(item), '=', item)
    
    dir = os.path.dirname(__file__)
    
    # copy libcom-api
    basename = os.path.basename(com_api)
    dest = os.path.join(dir, 'aaf', basename)
    print("copying", os.path.basename(com_api), '->', dest)
    shutil.copy(com_api, dest)
    
    # create ext dir
    aafext_dir = os.path.join(dir, 'aaf', 'aafext')
    if not os.path.exists(aafext_dir):
        print('creating', aafext_dir)
        os.makedirs(aafext_dir)
        
    # copy libaafintp<commit_msg>rename dir to dirpath<commit_after>from __future__ import print_function
from distutils.core import setup, Extension, Command 
from distutils.command.build_ext import build_ext
import os
import subprocess
import sys
import shutil
import glob
from Cython.Build import cythonize

#os.environ['CXX'] = 'g++'
#os.environ['ARCHFLAGS'] ="-arch x86_64"

# add this to fix build on mac osx mavericks
#os.environ['ARCHFLAGS'] ="-Wno-error=unused-command-line-argument-hard-error-in-future"

copy_args = sys.argv[1:]
AAF_ROOT = None
if '--aaf-root' in copy_args:
    index = copy_args.index('--aaf-root')
    AAF_ROOT = copy_args[index+1]
    del copy_args[index]
    del copy_args[index]
else:
    AAF_ROOT = os.environ.get("AAF_ROOT")
    
USE_AAF_SDK_DEBUG = bool(int(os.environ.get("USE_AAF_SDK_DEBUG", "1")))
NTHREADS= int(os.environ.get("NTHREADS",0))

space = '   '
if AAF_ROOT is None:

    print( space, "Unable to locate AAF Development libraries.")
    print( space, "Please specify with --aaf-root or AAF_ROOT env variable")
    print( space, "AAF SDK can be found from http://aaf.sourceforge.net")
    print( space, "Pre-built devel libraries can be found here")
    print( space, "http://sourceforge.net/projects/aaf/files/AAF-devel-libs/1.1.6")
    sys.exit(-1)
    
if not os.path.exists(AAF_ROOT):
    print(space, "AAF_ROOT direcotry does not exist: %s" % AAF_ROOT)
    sys.exit(-1)
    
AAF_INCLUDE = os.path.join(AAF_ROOT,'include')

AAF_LIB = os.path.join(AAF_ROOT,'lib', 'debug')

AAF_COM = os.path.join(AAF_ROOT,'bin', 'debug')

ext_extra = {
    'include_dirs': ['headers',AAF_INCLUDE],
    'library_dirs': [AAF_LIB, AAF_COM],
    'libraries': ['aaflib','aafiid', 'com-api'],
}

if sys.platform.startswith('linux'):
    ext_extra['extra_link_args'] = ['-Wl,-R$ORIGIN']

WIN_ARCH = 'Win32'

if sys.platform.startswith('win'):
    import platform
    if platform.architecture()[0] == '64bit':
        WIN_ARCH = 'x64'
    if '--debug' in copy_args or USE_AAF_SDK_DEBUG:
        ext_extra['library_dirs'] = [os.path.join(AAF_ROOT,WIN_ARCH ,'Debug','Refimpl')]
        ext_extra['libraries'] = ['AAFD', 'AAFIIDD']
    else:
        ext_extra['library_dirs'] = [os.path.join(AAF_ROOT,WIN_ARCH ,'Release','Refimpl')]
        ext_extra['libraries'] = ['AAF', 'AAFIID']

    ext_extra['library_dirs'].extend([os.path.join(AAF_ROOT, 'lib'),
                                        os.path.join(AAF_ROOT, 'bin')])
    
print("AAF_ROOT =",AAF_ROOT)


ext_modules = []
for dirname, dirnames, filenames in os.walk("aaf", topdown=True):
    for filename in filenames:
        if filename.startswith('.') or os.path.splitext(filename)[1] != '.pyx':
            continue

        path = os.path.join(dirname, filename)
        name = "aaf." + os.path.splitext(os.path.basename(path))[0]
        
        sources = [path]

        extra_src_dir = os.path.join(os.path.dirname(__file__), 'aaf', os.path.splitext(filename)[0])
        extra_src = glob.glob(os.path.join(extra_src_dir, '*.cpp'))
        
        sources.extend(extra_src)
        extension = Extension(name,
                              sources=sources,
                              language="c++",
                              **ext_extra)

        ext_modules.append(extension)
    break

def get_com_api(debug=True):
    if sys.platform.startswith("win"):
        dirpath = os.path.join(AAF_ROOT, '%s' % str(WIN_ARCH))
        if debug or USE_AAF_SDK_DEBUG:
            dirpath = os.path.join(dirpath, "Debug")
        else:
            dirpath = os.path.join(dirpath, "Release")

        for dirname in ext_extra['library_dirs']:
            com_api = os.path.join(dirname, 'AAFCOAPI.dll')
            libaafintp =  os.path.join(dirname, 'aafext', 'AAFINTP.dll')
            libaafpgapi =  os.path.join(dirname, 'aafext', 'AAFPGAPI.dll')
            #print(os.path.basename(com_api), '=', com_api)
            if all([os.path.exists(item) for item in (com_api,libaafintp,libaafpgapi)]):
                return com_api, libaafintp, libaafpgapi
        raise Exception("Unable to find AAFCOAPI.dll, AAFINTP.dll, AAFPGAPI.dll")
    ext = '.so'
    if sys.platform == 'darwin':
        ext = '.dylib'
    dirpath = os.path.join(AAF_ROOT, 'bin')
    if debug or USE_AAF_SDK_DEBUG:
        dirpath = os.path.join(dirpath, 'debug')
    
    com_api = os.path.join(dirpath, 'libcom-api' + ext)
    libaafintp = os.path.join(dirpath, 'aafext', 'libaafintp' + ext)
    libaafpgapi = os.path.join(dirpath, 'aafext', 'libaafpgapi' + ext)
    
    return com_api, libaafintp, libaafpgapi

def copy_com_api(debug=True):
    com_api, libaafintp, libaafpgapi = get_com_api(debug)
    
    for item in [com_api, libaafintp, libaafpgapi]:
        print(os.path.basename(item), '=', item)
    
    dirpath = os.path.dirname(__file__)
    
    # copy libcom-api
    basename = os.path.basename(com_api)
    dest = os.path.join(dirpath, 'aaf', basename)
    print("copying", os.path.basename(com_api), '->', dest)
    shutil.copy(com_api, dest)
    
    # create ext dir
    aafext_dir = os.path.join(dirpath, 'aaf', 'aafext')
    if not os.path.exists(aafext_dir):
        print('creating', aafext_dir)
        os.makedirs(aafext_dir)
        
    # copy libaafintp
{'fraction_non_alphanumeric': 0.14824823858700897, 'fraction_numerical': 0.004439725895183862, 'mean_word_length': 2.9838523644752017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5020372', 'n_tokens_mistral': 3887, 'n_tokens_neox': 3831, 'n_words': 711}