<commit_before>"""
The Compiler uses the Parser and the Kernel to translate a pure Python function
into a native library. This library can be invoked though several language bindings,
in particular as a native Python module.
"""


import subprocess
from vecpy.kernel import *
from vecpy.compiler_constants import *
from vecpy.compiler_generic import Compiler_Generic
from vecpy.compiler_intel import Compiler_Intel

class Compiler:

  #Utility functions: output file names
  def get_python_file(k):
    return 'vecpy_%s_python.h'%(k.name)

  def get_java_file(k):
    return 'vecpy_%s_java.h'%(k.name)

  def get_cpp_file(k):
    return 'vecpy_%s_cpp.h'%(k.name)

  def get_kernel_file(k):
    return 'vecpy_%s_kernel.h'%(k.name)

  def get_core_file(k):
    return 'vecpy_%s_core.cpp'%(k.name)

  #Generates the core file
  def compile_core(k, options, include_files):
    if Architecture.is_generic(options.arch):
      suffix = 'scalar'
    elif Architecture.is_intel(options.arch):
      suffix = 'vector'
    else:
      raise Exception('Target architecture not implemented (%s)'%(options.arch['name']))
    src = Formatter()
    src.section('VecPy generated core')
    #Includes
    src += '//Includes'
    src += '#include <pthread.h>'
    src += '#include <stdio.h>'
    src += '#include "%s"'%(Compiler.get_kernel_file(k))
    src += ''
    #Utility functions
    src += '//Utility functions'
    src += 'static void* threadStart(void* v) {'
    src.indent()
    src += '%s_%s((KernelArgs*)v);'%(k.name, suffix)
    src += 'return NULL;'
    src.unindent()
    src += '}'
    src += 'static bool isAligned(void* data) {'
    src.indent()
    src += 'return reinterpret_cast<unsigned long>(data) %% %dUL == 0UL;'%(options.arch['size'] * 4)
    src.unindent()
    src += '}'
    src += 'static bool checkArgs(KernelArgs* args) {'
    src.indent()
    for arg in k.get_arguments(uniform=False):
      src += 'if(!isAligned(args->%s)) {'%(arg.name)
      src.indent()
      src += 'printf("Array not aligned (%s)\\n");'%(arg.name)
      src += 'return false;'
      src.unindent()
      src += '}'
    src += 'return true;'
    src.unindent()
    src += '}'
    src += ''
    #Unified core for all programming interfaces
    src += '//Unified core function'
    src += 'static bool run(KernelArgs* args) {'
    src.indent()
    src += 'if(!checkArgs(args)) {'
    src.indent()
    src += 'printf("Arguments are invalid\\n");'
    src += 'return false;'
    src.unindent()
    src += '}'
    src += '//Compile-time constants'
    src += 'const unsigned int vectorSize = %d;'%(options.arch['size'])
    src += 'const unsigned int numThreads = %d;'%(options.threads)
    src += '//Division of labor'
    src += 'const unsigned int vectorsPerThread = args->N / (vectorSize * numThreads);'
    src += 'const unsigned int elementsPerThread = vectorsPerThread * vectorSize;'
    src += '//Execute on multiple threads'
    src += 'unsigned int offset = 0;'
    src += 'if(elementsPerThread > 0) {'
    src.indent()
    src += 'pthread_t* threads = new pthread_t[numThreads];'
    src += 'KernelArgs* threadArgs = new KernelArgs[numThreads];'
    src += 'for(int t = 0; t < numThreads; t++) {'
    src.indent()
    for arg in k.get_arguments():
      if arg.is_uniform:
        src += 'threadArgs[t].%s = args->%s;'%(arg.name, arg.name)
      else:
        offset = 'offset'
        if arg.stride > 1:
          offset = '%s * %d'%(offset, arg.stride)
        src += 'threadArgs[t].%s = &args->%s[%s];'%(arg.name, arg.name, offset)
    src += 'threadArgs[t].N = elementsPerThread;'
    src += 'offset += elementsPerThread;'
    src += 'pthread_create(&threads[t], NULL, threadStart, (void*)&threadArgs[t]);'
    src.unindent()
    src += '}'
    src += 'for(int t = 0; t < numThreads; t++) {'
    src.indent()
    src += ' pthread_join(threads[t], NULL); '
    src.unindent()
    src += '}'
    src += 'delete [] threads;'
    src += 'delete [] threadArgs;'
    src.unindent()
    src += '}'
    #src += 'printf("Vector: %dx%dx%d=%d | Scalar: %d\\n", vectorsPerThread, vectorSize, numThreads, offset, args->N - offset);'
    src += '//Handle any remaining elements'
    src += 'if(offset < args->N) {'
    src.indent()
    src += 'KernelArgs lastArgs;'
    for arg in k.get_arguments():
      if arg.is_uniform:
        src += 'lastArgs.%s = args->%s;'%(arg.name, arg.name)
      else:
        offset = 'offset'
        if arg.stride > 1:
          offset = '%s * %d'%(offset, arg.stride)
        src += 'lastArgs.%s = &args->%s[%s];'%(arg.name, arg.name, offset)
    src += 'lastArgs.N = args->N - offset;'
    src += '%s_scalar(&lastArgs);'%(k.name)
    src.unindent()
    src += '}'
    src += 'return true;'
    src.unindent()
    src += '}'
    src += ''
    #Additional includes for each programming language
    src += '//Additional includes for each programming language'
    for file in include_files:
      src += '#include "%s"'%(file)
    src += ''
    #Save code to file
    file_name = Compiler.get_core_file(k)
    with open(file_name, 'w') as file:
      file.write(src.get_code())
    #print('Saved to file: %s'%(file_name))

  #Generates the C++ API
  def compile_cpp(k, options):
    src = Formatter()
    src.section('VecPy generated entry point: C++')
    #Build the argument string
    arg_str = ''
    for arg in k.get_arguments():
      arg_str += '%s%s %s, '%(options.type, '*' if not arg.is_uniform else '', arg.name)
    #Wrapper for the core function
    src += '//Wrapper for the core function'
    src += 'extern "C" bool %s(%sint N) {'%(k.name, arg_str)
    src.indent()
    src += 'KernelArgs args;'
    for arg in k.get_arguments():
      src += 'args.%s = %s;'%(arg.name, arg.name)
    src += 'args.N = N;'
    src += 'return run(&args);'
    src.unindent()
    src += '}'
    src += ''
    #Save code to file
    file_name = Compiler.get_cpp_file(k)
    with open(file_name, 'w') as file:
      file.write(src.get_code())
    #print('Saved to file: %s'%(file_name))

  #Generates the Python API
  def compile_python(k, options):
    type = options.type
    module_name = 'VecPy_' + k.name
    args = k.get_arguments()
    if DataType.is_floating(type):
      uniform_pytype = 'f'
      uniform_ctype = 'float'
    elif DataType.is_integral(type):
      uniform_pytype = 'I'
      uniform_ctype = 'unsigned int'
    else:
      raise Exception('Unsupported data type (%s)'%(type))
    src = Formatter()
    src.section('VecPy generated entry point: Python')
    #Includes
    src += '//Includes'
    src += '#include <Python.h>'
    src += ''
    #Wrapper for the core function
    src += '//Wrapper for the core function'
    src += 'static PyObject* %s_run(PyObject* self, PyObject* pyArgs) {'%(k.name)
    src.indent()
    src += '//Handles to Python objects and buffers'
    obj_str = ', '.join('*obj_%s'%(arg.name) for arg in k.get_arguments(uniform=False))
    src += 'PyObject %s;'%(obj_str)
    if len(k.get_arguments(uniform=True)) > 0:
      obj_str = ', '.join('vp_%s'%(arg.name) for arg in k.get_arguments(uniform=True))
      src += '%s %s;'%(uniform_ctype, obj_str)
    buf_str = ', '.join('vp_%s'%(arg.name) for arg in k.get_arguments(uniform=False))
    src += 'Py_buffer %s;'%(buf_str)
    src += '//Get Python objects'
    obj_str = ', '.join('&%s_%s'%('vp' if arg.is_uniform else 'obj', arg.name) for arg in args)
    decode_str = ''.join(uniform_pytype if arg.is_uniform else 'O' for arg in args)<commit_msg>Fixed Python module entry point name.<commit_after>"""
The Compiler uses the Parser and the Kernel to translate a pure Python function
into a native library. This library can be invoked though several language bindings,
in particular as a native Python module.
"""


import subprocess
from vecpy.kernel import *
from vecpy.compiler_constants import *
from vecpy.compiler_generic import Compiler_Generic
from vecpy.compiler_intel import Compiler_Intel

class Compiler:

  #Utility functions: output file names
  def get_python_file(k):
    return 'vecpy_%s_python.h'%(k.name)

  def get_java_file(k):
    return 'vecpy_%s_java.h'%(k.name)

  def get_cpp_file(k):
    return 'vecpy_%s_cpp.h'%(k.name)

  def get_kernel_file(k):
    return 'vecpy_%s_kernel.h'%(k.name)

  def get_core_file(k):
    return 'vecpy_%s_core.cpp'%(k.name)

  #Generates the core file
  def compile_core(k, options, include_files):
    if Architecture.is_generic(options.arch):
      suffix = 'scalar'
    elif Architecture.is_intel(options.arch):
      suffix = 'vector'
    else:
      raise Exception('Target architecture not implemented (%s)'%(options.arch['name']))
    src = Formatter()
    src.section('VecPy generated core')
    #Includes
    src += '//Includes'
    src += '#include <pthread.h>'
    src += '#include <stdio.h>'
    src += '#include "%s"'%(Compiler.get_kernel_file(k))
    src += ''
    #Utility functions
    src += '//Utility functions'
    src += 'static void* threadStart(void* v) {'
    src.indent()
    src += '%s_%s((KernelArgs*)v);'%(k.name, suffix)
    src += 'return NULL;'
    src.unindent()
    src += '}'
    src += 'static bool isAligned(void* data) {'
    src.indent()
    src += 'return reinterpret_cast<unsigned long>(data) %% %dUL == 0UL;'%(options.arch['size'] * 4)
    src.unindent()
    src += '}'
    src += 'static bool checkArgs(KernelArgs* args) {'
    src.indent()
    for arg in k.get_arguments(uniform=False):
      src += 'if(!isAligned(args->%s)) {'%(arg.name)
      src.indent()
      src += 'printf("Array not aligned (%s)\\n");'%(arg.name)
      src += 'return false;'
      src.unindent()
      src += '}'
    src += 'return true;'
    src.unindent()
    src += '}'
    src += ''
    #Unified core for all programming interfaces
    src += '//Unified core function'
    src += 'static bool run(KernelArgs* args) {'
    src.indent()
    src += 'if(!checkArgs(args)) {'
    src.indent()
    src += 'printf("Arguments are invalid\\n");'
    src += 'return false;'
    src.unindent()
    src += '}'
    src += '//Compile-time constants'
    src += 'const unsigned int vectorSize = %d;'%(options.arch['size'])
    src += 'const unsigned int numThreads = %d;'%(options.threads)
    src += '//Division of labor'
    src += 'const unsigned int vectorsPerThread = args->N / (vectorSize * numThreads);'
    src += 'const unsigned int elementsPerThread = vectorsPerThread * vectorSize;'
    src += '//Execute on multiple threads'
    src += 'unsigned int offset = 0;'
    src += 'if(elementsPerThread > 0) {'
    src.indent()
    src += 'pthread_t* threads = new pthread_t[numThreads];'
    src += 'KernelArgs* threadArgs = new KernelArgs[numThreads];'
    src += 'for(int t = 0; t < numThreads; t++) {'
    src.indent()
    for arg in k.get_arguments():
      if arg.is_uniform:
        src += 'threadArgs[t].%s = args->%s;'%(arg.name, arg.name)
      else:
        offset = 'offset'
        if arg.stride > 1:
          offset = '%s * %d'%(offset, arg.stride)
        src += 'threadArgs[t].%s = &args->%s[%s];'%(arg.name, arg.name, offset)
    src += 'threadArgs[t].N = elementsPerThread;'
    src += 'offset += elementsPerThread;'
    src += 'pthread_create(&threads[t], NULL, threadStart, (void*)&threadArgs[t]);'
    src.unindent()
    src += '}'
    src += 'for(int t = 0; t < numThreads; t++) {'
    src.indent()
    src += ' pthread_join(threads[t], NULL); '
    src.unindent()
    src += '}'
    src += 'delete [] threads;'
    src += 'delete [] threadArgs;'
    src.unindent()
    src += '}'
    #src += 'printf("Vector: %dx%dx%d=%d | Scalar: %d\\n", vectorsPerThread, vectorSize, numThreads, offset, args->N - offset);'
    src += '//Handle any remaining elements'
    src += 'if(offset < args->N) {'
    src.indent()
    src += 'KernelArgs lastArgs;'
    for arg in k.get_arguments():
      if arg.is_uniform:
        src += 'lastArgs.%s = args->%s;'%(arg.name, arg.name)
      else:
        offset = 'offset'
        if arg.stride > 1:
          offset = '%s * %d'%(offset, arg.stride)
        src += 'lastArgs.%s = &args->%s[%s];'%(arg.name, arg.name, offset)
    src += 'lastArgs.N = args->N - offset;'
    src += '%s_scalar(&lastArgs);'%(k.name)
    src.unindent()
    src += '}'
    src += 'return true;'
    src.unindent()
    src += '}'
    src += ''
    #Additional includes for each programming language
    src += '//Additional includes for each programming language'
    for file in include_files:
      src += '#include "%s"'%(file)
    src += ''
    #Save code to file
    file_name = Compiler.get_core_file(k)
    with open(file_name, 'w') as file:
      file.write(src.get_code())
    #print('Saved to file: %s'%(file_name))

  #Generates the C++ API
  def compile_cpp(k, options):
    src = Formatter()
    src.section('VecPy generated entry point: C++')
    #Build the argument string
    arg_str = ''
    for arg in k.get_arguments():
      arg_str += '%s%s %s, '%(options.type, '*' if not arg.is_uniform else '', arg.name)
    #Wrapper for the core function
    src += '//Wrapper for the core function'
    src += 'extern "C" bool %s(%sint N) {'%(k.name, arg_str)
    src.indent()
    src += 'KernelArgs args;'
    for arg in k.get_arguments():
      src += 'args.%s = %s;'%(arg.name, arg.name)
    src += 'args.N = N;'
    src += 'return run(&args);'
    src.unindent()
    src += '}'
    src += ''
    #Save code to file
    file_name = Compiler.get_cpp_file(k)
    with open(file_name, 'w') as file:
      file.write(src.get_code())
    #print('Saved to file: %s'%(file_name))

  #Generates the Python API
  def compile_python(k, options):
    type = options.type
    module_name = 'vecpy_' + k.name
    args = k.get_arguments()
    if DataType.is_floating(type):
      uniform_pytype = 'f'
      uniform_ctype = 'float'
    elif DataType.is_integral(type):
      uniform_pytype = 'I'
      uniform_ctype = 'unsigned int'
    else:
      raise Exception('Unsupported data type (%s)'%(type))
    src = Formatter()
    src.section('VecPy generated entry point: Python')
    #Includes
    src += '//Includes'
    src += '#include <Python.h>'
    src += ''
    #Wrapper for the core function
    src += '//Wrapper for the core function'
    src += 'static PyObject* %s_run(PyObject* self, PyObject* pyArgs) {'%(k.name)
    src.indent()
    src += '//Handles to Python objects and buffers'
    obj_str = ', '.join('*obj_%s'%(arg.name) for arg in k.get_arguments(uniform=False))
    src += 'PyObject %s;'%(obj_str)
    if len(k.get_arguments(uniform=True)) > 0:
      obj_str = ', '.join('vp_%s'%(arg.name) for arg in k.get_arguments(uniform=True))
      src += '%s %s;'%(uniform_ctype, obj_str)
    buf_str = ', '.join('vp_%s'%(arg.name) for arg in k.get_arguments(uniform=False))
    src += 'Py_buffer %s;'%(buf_str)
    src += '//Get Python objects'
    obj_str = ', '.join('&%s_%s'%('vp' if arg.is_uniform else 'obj', arg.name) for arg in args)
    decode_str = ''.join(uniform_pytype if arg.is_uniform else 'O' for arg in args)
{'fraction_non_alphanumeric': 0.17074966532797858, 'fraction_numerical': 0.0012048192771084338, 'mean_word_length': 3.0567472169427097, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1939308', 'n_tokens_mistral': 5210, 'n_tokens_neox': 5072, 'n_words': 1310}