<commit_before>#if OCCA_COI_ENABLED

#include "occaCOI.hpp"

namespace occa {
  //---[ Kernel ]---------------------
  template <>
  kernel_t<COI>::kernel_t(){
    data = NULL;
    dev  = NULL;

    functionName = "";

    dims  = 1;
    inner = occa::dim(1,1,1);
    outer = occa::dim(1,1,1);

    preferredDimSize_ = 0;

    startTime = (void*) new coiEvent;
    endTime   = (void*) new coiEvent;
  }

  template <>
  kernel_t<COI>::kernel_t(const kernel_t<COI> &k){
    data = k.data;
    dev  = k.dev;

    functionName = k.functionName;

    dims  = k.dims;
    inner = k.inner;
    outer = k.outer;

    preferredDimSize_ = k.preferredDimSize_;

    startTime = k.startTime;
    endTime   = k.endTime;
  }

  template <>
  kernel_t<COI>& kernel_t<COI>::operator = (const kernel_t<COI> &k){
    data = k.data;
    dev  = k.dev;

    functionName = k.functionName;

    dims  = k.dims;
    inner = k.inner;
    outer = k.outer;

    preferredDimSize_ = k.preferredDimSize_;

    *((coiEvent*) startTime) = *((coiEvent*) k.startTime);
    *((coiEvent*) endTime)   = *((coiEvent*) k.endTime);

    return *this;
  }

  template <>
  kernel_t<COI>::~kernel_t(){}

  template <>
  kernel_t<COI>* kernel_t<COI>::buildFromSource(const std::string &filename,
                                                          const std::string &functionName_,
                                                          const kernelInfo &info_){
    functionName = functionName_;

    kernelInfo info = info_;
    info.addDefine("OCCA_USING_CPU", 1);
    info.addDefine("OCCA_USING_COI", 1);

    info.addOCCAKeywords(occaCOIDefines);

    std::stringstream salt;
    salt << "COI"
         << info.salt()
         << functionName;

    std::string cachedBinary = binaryIsCached(filename, salt.str());

    struct stat buffer;
    bool fileExists = (stat(cachedBinary.c_str(), &buffer) == 0);

    if(fileExists){
      std::cout << "Found cached binary of [" << filename << "] in [" << cachedBinary << "]\n";
      return buildFromBinary(cachedBinary, functionName);
    }

    data = new COIKernelData_t;

    std::string iCachedBinary = createIntermediateSource(filename,
                                                         cachedBinary,
                                                         info);

    std::stringstream command;

    command << dev->dHandle->compiler
            << " -o " << cachedBinary
            << " -x c++ -w -nodefaultlibs -shared -fPIC"
            << ' '    << dev->dHandle->compilerFlags
            << ' '    << info.flags
            << ' '    << iCachedBinary;

    const std::string &sCommand = command.str();

    std::cout << sCommand << '\n';

    system(sCommand.c_str());

    OCCA_EXTRACT_DATA(COI, Kernel);

    COILIBRARY outLibrary;

    OCCA_COI_CHECK("Kernel: Loading Kernel To Chief",
                   COIProcessLoadLibraryFromFile(data_.chiefID,
                                                 cachedBinary.c_str(),
                                                 NULL,
                                                 NULL,
                                                 &outLibrary));

    const char *c_functionName = functionName.c_str();

    OCCA_COI_CHECK("Kernel: Getting Handle",
                   COIProcessGetFunctionHandles(data_.chiefID,
                                                1,
                                                &c_functionName,
                                                &(data_.kernel)));

    return this;
  }

  template <>
  kernel_t<COI>* kernel_t<COI>::buildFromBinary(const std::string &filename,
                                                const std::string &functionName_){
    data = new COIKernelData_t;

    OCCA_EXTRACT_DATA(COI, Kernel);

    functionName = functionName_;

    COILIBRARY outLibrary;

    OCCA_COI_CHECK("Kernel: Loading Kernel To Chief",
                   COIProcessLoadLibraryFromFile(data_.chiefID,
                                                 NULL,
                                                 filename.c_str(),
                                                 NULL,
                                                 &outLibrary));

    const char *c_functionName = functionName.c_str();

    OCCA_COI_CHECK("Kernel: Getting Handle",
                   COIProcessGetFunctionHandles(data_.chiefID,
                                                1,
                                                &c_functionName,
                                                &(data_.kernel)));

    return this;
  }

  // [-] Missing
  template <>
  int kernel_t<COI>::preferredDimSize(){
    preferredDimSize_ = 1;

    return preferredDimSize_;
  }

  OCCA_COI_KERNEL_OPERATOR_DEFINITIONS;

  template <>
  double kernel_t<COI>::timeTaken(){
    const double &start = *((double*) startTime);
    const double &end   = *((double*) endTime);

    return 1.0e3*(end - start);
  }

  template <>
  void kernel_t<COI>::free(){
  }
  //==================================


  //---[ Memory ]---------------------
  template <>
  memory_t<COI>::memory_t(){
    handle = NULL;
    dev    = NULL;
    size = 0;
  }

  template <>
  memory_t<COI>::memory_t(const memory_t<COI> &m){
    handle = m.handle;
    dev    = m.dev;
    size   = m.size;
  }

  template <>
  memory_t<COI>& memory_t<COI>::operator = (const memory_t<COI> &m){
    handle = m.handle;
    dev    = m.dev;
    size   = m.size;

    return *this;
  }

  template <>
  memory_t<COI>::~memory_t(){}

  template <>
  void memory_t<COI>::copyFrom(const void *source,
                               const size_t bytes,
                               const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferWrite(*((coiMemory*) handle),
                                  offset,
                                  source,
                                  bytes,
                                  COI_COPY_UNSPECIFIED,
                                  waitingOnEvent,
                                  lastEventPtr,
                                  stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyFrom(const memory_v *source,
                               const size_t bytes,
                               const size_t destOffset,
                               const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <=         size);
    OCCA_CHECK((bytes_ + srcOffset)  <= source->size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) handle),
                                 *((coiMemory*) source->handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyTo(void *dest,
                             const size_t bytes,
                             const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferRead(*((coiMemory*) handle),
                                 offset,
                                 dest,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyTo(memory_v *dest,
                             const size_t bytes,
                             const size_t destOffset,
                             const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <= dest->size);
    OCCA_CHECK((bytes_ + srcOffset)  <=       size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) dest->handle),
                                 *((coiMemory*) handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::asyncCopyFrom(const void *source,
                                    const size_t bytes,
                                    const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferWrite(*((coiMemory*) handle),
                                  offset,
                                  source,
                                  bytes,
                                  COI_COPY_UNSPECIFIED,
                                  waitingOnEvent,
                                  lastEventPtr,
                                  stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyFrom(const memory_v *source,
                                    const size_t bytes,
                                    const size_t destOffset,
                                    const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <=         size);
    OCCA_CHECK((bytes_ + srcOffset)  <= source->size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) handle),
                                 *((coiMemory*) source->handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyTo(void *dest,
                                  const size_t bytes,
                                  const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferRead(*((coiMemory*) handle),
                                 offset,
                                 dest,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyTo(memory_v *dest,
                                  const size_t bytes,
                                  const size_t destOffset,
                                  const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <= dest->size);
    OCCA_CHECK((bytes_ + srcOffset)  <=       size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) dest->handle),
                                 *((coiMemory*) handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::free(){
    OCCA_COI_CHECK("Memory: free",
                   COIBufferDestroy( *((coiMemory*) handle) ) );

    ::free(handle);
  }
  //==================================


  //---[ Device ]---------------------
  template <>
  device_t<COI>::device_t() :
    memoryUsed(0) {
    data = NULL;

    getEnvironmentVariables();
  }

  template <>
  device_t<COI>::device_t(int platform, int device) :
    memoryUsed(0) {
    data = NULL;

    getEnvironmentVariables();
  }

  template <>
  device_t<COI>::device_t(const device_t<COI> &d){
    data       = d.data;
    memoryUsed = d.memoryUsed;

    compilerFlags = d.compilerFlags;
  }

  template <>
  device_t<COI>& device_t<COI>::operator = (const device_t<COI> &d){
    data       = d.data;
    memoryUsed = d.memoryUsed;

    compilerFlags = d.compilerFlags;

    return *this;
  }

  template <>
  void device_t<COI>::setup(const int device, const int memoryAllocated){
    data = ::malloc(sizeof(COIDeviceData_t));

    OCCA_EXTRACT_DATA(COI, Device);

    uint32_t deviceCount;
    OCCA_COI_CHECK("Device: Get Count",
                   COIEngineGetCount(COI_ISA_MIC, &deviceCount));

    OCCA_CHECK(device < deviceCount);

    OCCA_COI_CHECK("Device: Get Handle",
                   COIEngineGetHandle(COI_ISA_MIC, device, &data_.deviceID) );

    std::stringstream salt;
    salt << "COI"
         << occaCOIMain;

    std::string cachedBinary = binaryIsCached("occaCOIMain", salt.str());

    struct stat buffer;
    bool fileExists = (stat(cachedBinary.c_str(), &buffer) == 0);

    if(fileExists)
      std::cout << "Found cached binary of [occaCOIMain] in [" << cachedBinary << "]\n";
    else{
      //---[ Write File ]-----------------
      int lastSlash = 0;
      const int chars = cachedBinary.size();

      for(int i = 0; i < chars; ++i)
        if(cachedBinary[i] == '/')
          lastSlash = i;

      ++lastSlash;

      const std::string iCachedBinary =
        cachedBinary.substr(0, lastSlash) + "i_" + cachedBinary.substr(lastSlash, chars);

      std::ofstream fs;
      fs.open(iCachedBinary.c_str());

      fs << occaCOIMain;

      fs.close();

      std::stringstream command;

      command << dev->dHandle->compiler
              << " -o " << cachedBinary
              << " -x c++"
              << ' '    << dev->dHandle->compilerFlags
              << ' '    << iCachedBinary;

      const std::string &sCommand = command.str();

      std::cout << sCommand << '\n';

      system(sCommand.c_str());
    }

    OCCA_COI_CHECK("Device: Initializing",
                   COIProcessCreateFromFile(data_.deviceID,
                                            cachedBinary.c_str(),
                                            0    , NULL,
                                            false, NULL,
                                            true , NULL,
                                            memoryAllocated ? memoryAllocated : (4 << 30), // 4 GB
                                            NULL,
                                            &data_.chiefID) );<commit_msg>COI: Had a stray [new] that was messing things up...<commit_after>#if OCCA_COI_ENABLED

#include "occaCOI.hpp"

namespace occa {
  //---[ Kernel ]---------------------
  template <>
  kernel_t<COI>::kernel_t(){
    data = NULL;
    dev  = NULL;

    functionName = "";

    dims  = 1;
    inner = occa::dim(1,1,1);
    outer = occa::dim(1,1,1);

    preferredDimSize_ = 0;

    startTime = (void*) new coiEvent;
    endTime   = (void*) new coiEvent;
  }

  template <>
  kernel_t<COI>::kernel_t(const kernel_t<COI> &k){
    data = k.data;
    dev  = k.dev;

    functionName = k.functionName;

    dims  = k.dims;
    inner = k.inner;
    outer = k.outer;

    preferredDimSize_ = k.preferredDimSize_;

    startTime = k.startTime;
    endTime   = k.endTime;
  }

  template <>
  kernel_t<COI>& kernel_t<COI>::operator = (const kernel_t<COI> &k){
    data = k.data;
    dev  = k.dev;

    functionName = k.functionName;

    dims  = k.dims;
    inner = k.inner;
    outer = k.outer;

    preferredDimSize_ = k.preferredDimSize_;

    *((coiEvent*) startTime) = *((coiEvent*) k.startTime);
    *((coiEvent*) endTime)   = *((coiEvent*) k.endTime);

    return *this;
  }

  template <>
  kernel_t<COI>::~kernel_t(){}

  template <>
  kernel_t<COI>* kernel_t<COI>::buildFromSource(const std::string &filename,
                                                          const std::string &functionName_,
                                                          const kernelInfo &info_){
    functionName = functionName_;

    kernelInfo info = info_;
    info.addDefine("OCCA_USING_CPU", 1);
    info.addDefine("OCCA_USING_COI", 1);

    info.addOCCAKeywords(occaCOIDefines);

    std::stringstream salt;
    salt << "COI"
         << info.salt()
         << functionName;

    std::string cachedBinary = binaryIsCached(filename, salt.str());

    int lastSlash = 0;
    const int chars = cachedBinary.size();

    for(int i = 0; i < chars; ++i)
      if(cachedBinary[i] == '/')
        lastSlash = i;

    ++lastSlash;

    std::string soname  = cachedBinary.substr(lastSlash, chars);
    std::string libPath = cachedBinary.substr(0, lastSlash);
    std::string libName = "lib" + soname + ".so";

    cachedBinary = libPath + libName;

    struct stat buffer;
    bool fileExists = (stat(cachedBinary.c_str(), &buffer) == 0);

    if(fileExists){
      std::cout << "Found cached binary of [" << filename << "] in [" << cachedBinary << "]\n";
      return buildFromBinary(cachedBinary, functionName);
    }

    std::string iCachedBinary = createIntermediateSource(filename,
                                                         cachedBinary,
                                                         info);

    std::stringstream command;

    command << dev->dHandle->compiler
            << " -o " << cachedBinary
            << " -x c++ -w -nodefaultlibs -shared -fPIC"
            << ' '    << dev->dHandle->compilerFlags
            << ' '    << info.flags
            << ' '    << iCachedBinary;

    const std::string &sCommand = command.str();

    std::cout << sCommand << '\n';

    system(sCommand.c_str());

    OCCA_EXTRACT_DATA(COI, Kernel);

    COILIBRARY outLibrary;

    OCCA_COI_CHECK("Kernel: Loading Kernel To Chief",
                   COIProcessLoadLibraryFromFile(data_.chiefID,
                                                 cachedBinary.c_str(),
                                                 soname.c_str(),
                                                 NULL,
                                                 &outLibrary));

    const char *c_functionName = functionName.c_str();

    OCCA_COI_CHECK("Kernel: Getting Handle",
                   COIProcessGetFunctionHandles(data_.chiefID,
                                                1,
                                                &c_functionName,
                                                &(data_.kernel)));

    return this;
  }

  template <>
  kernel_t<COI>* kernel_t<COI>::buildFromBinary(const std::string &filename,
                                                const std::string &functionName_){
    OCCA_EXTRACT_DATA(COI, Kernel);

    functionName = functionName_;

    COILIBRARY outLibrary;

    OCCA_COI_CHECK("Kernel: Loading Kernel To Chief",
                   COIProcessLoadLibraryFromFile(data_.chiefID,
                                                 NULL,
                                                 filename.c_str(),
                                                 NULL,
                                                 &outLibrary));

    const char *c_functionName = functionName.c_str();

    OCCA_COI_CHECK("Kernel: Getting Handle",
                   COIProcessGetFunctionHandles(data_.chiefID,
                                                1,
                                                &c_functionName,
                                                &(data_.kernel)));

    return this;
  }

  // [-] Missing
  template <>
  int kernel_t<COI>::preferredDimSize(){
    preferredDimSize_ = 1;

    return preferredDimSize_;
  }

  OCCA_COI_KERNEL_OPERATOR_DEFINITIONS;

  template <>
  double kernel_t<COI>::timeTaken(){
    const double &start = *((double*) startTime);
    const double &end   = *((double*) endTime);

    return 1.0e3*(end - start);
  }

  template <>
  void kernel_t<COI>::free(){
  }
  //==================================


  //---[ Memory ]---------------------
  template <>
  memory_t<COI>::memory_t(){
    handle = NULL;
    dev    = NULL;
    size = 0;
  }

  template <>
  memory_t<COI>::memory_t(const memory_t<COI> &m){
    handle = m.handle;
    dev    = m.dev;
    size   = m.size;
  }

  template <>
  memory_t<COI>& memory_t<COI>::operator = (const memory_t<COI> &m){
    handle = m.handle;
    dev    = m.dev;
    size   = m.size;

    return *this;
  }

  template <>
  memory_t<COI>::~memory_t(){}

  template <>
  void memory_t<COI>::copyFrom(const void *source,
                               const size_t bytes,
                               const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferWrite(*((coiMemory*) handle),
                                  offset,
                                  source,
                                  bytes,
                                  COI_COPY_UNSPECIFIED,
                                  waitingOnEvent,
                                  lastEventPtr,
                                  stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyFrom(const memory_v *source,
                               const size_t bytes,
                               const size_t destOffset,
                               const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <=         size);
    OCCA_CHECK((bytes_ + srcOffset)  <= source->size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) handle),
                                 *((coiMemory*) source->handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyTo(void *dest,
                             const size_t bytes,
                             const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferRead(*((coiMemory*) handle),
                                 offset,
                                 dest,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::copyTo(memory_v *dest,
                             const size_t bytes,
                             const size_t destOffset,
                             const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <= dest->size);
    OCCA_CHECK((bytes_ + srcOffset)  <=       size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) dest->handle),
                                 *((coiMemory*) handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );

    OCCA_COI_CHECK("Memory: Blocking on Memory Transfer",
                   COIEventWait(1, stream.lastEvent,
                                -1, true, NULL, NULL) );
  }

  template <>
  void memory_t<COI>::asyncCopyFrom(const void *source,
                                    const size_t bytes,
                                    const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferWrite(*((coiMemory*) handle),
                                  offset,
                                  source,
                                  bytes,
                                  COI_COPY_UNSPECIFIED,
                                  waitingOnEvent,
                                  lastEventPtr,
                                  stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyFrom(const memory_v *source,
                                    const size_t bytes,
                                    const size_t destOffset,
                                    const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <=         size);
    OCCA_CHECK((bytes_ + srcOffset)  <= source->size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) handle),
                                 *((coiMemory*) source->handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyTo(void *dest,
                                  const size_t bytes,
                                  const size_t offset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + offset) <= size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferRead(*((coiMemory*) handle),
                                 offset,
                                 dest,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::asyncCopyTo(memory_v *dest,
                                  const size_t bytes,
                                  const size_t destOffset,
                                  const size_t srcOffset){
    coiStream &stream = *((coiStream*) dev->currentStream);

    const size_t bytes_ = (bytes == 0) ? size : bytes;

    OCCA_CHECK((bytes_ + destOffset) <= dest->size);
    OCCA_CHECK((bytes_ + srcOffset)  <=       size);

    bool waitingOnEvent    = (stream.lastEvent != NULL);
    coiEvent *lastEventPtr = stream.lastEvent;

    OCCA_COI_CHECK("Memory: Copy From",
                   COIBufferCopy(*((coiMemory*) dest->handle),
                                 *((coiMemory*) handle),
                                 destOffset,
                                 srcOffset,
                                 bytes,
                                 COI_COPY_UNSPECIFIED,
                                 waitingOnEvent,
                                 lastEventPtr,
                                 stream.lastEvent) );
  }

  template <>
  void memory_t<COI>::free(){
    OCCA_COI_CHECK("Memory: free",
                   COIBufferDestroy( *((coiMemory*) handle) ) );

    ::free(handle);
  }
  //==================================


  //---[ Device ]---------------------
  template <>
  device_t<COI>::device_t() :
    memoryUsed(0) {
    data = NULL;

    getEnvironmentVariables();
  }

  template <>
  device_t<COI>::device_t(int platform, int device) :
    memoryUsed(0) {
    data = NULL;

    getEnvironmentVariables();
  }

  template <>
  device_t<COI>::device_t(const device_t<COI> &d){
    data       = d.data;
    memoryUsed = d.memoryUsed;

    compilerFlags = d.compilerFlags;
  }

  template <>
  device_t<COI>& device_t<COI>::operator = (const device_t<COI> &d){
    data       = d.data;
    memoryUsed = d.memoryUsed;

    compilerFlags = d.compilerFlags;

    return *this;
  }

  template <>
  void device_t<COI>::setup(const int device, const int memoryAllocated){
    data = ::malloc(sizeof(COIDeviceData_t));

    OCCA_EXTRACT_DATA(COI, Device);

    uint32_t deviceCount;
    OCCA_COI_CHECK("Device: Get Count",
                   COIEngineGetCount(COI_ISA_MIC, &deviceCount));

    OCCA_CHECK(device < deviceCount);

    OCCA_COI_CHECK("Device: Get Handle",
                   COIEngineGetHandle(COI_ISA_MIC, device, &data_.deviceID) );

    std::stringstream salt;
    salt << "COI"
         << occaCOIMain;

    std::string cachedBinary = binaryIsCached("occaCOIMain", salt.str());

    struct stat buffer;
    bool fileExists = (stat(cachedBinary.c_str(), &buffer) == 0);

    if(fileExists)
      std::cout << "Found cached binary of [occaCOIMain] in [" << cachedBinary << "]\n";
    else{
      //---[ Write File ]-----------------
      int lastSlash = 0;
      const int chars = cachedBinary.size();

      for(int i = 0; i < chars; ++i)
        if(cachedBinary[i] == '/')
          lastSlash = i;

      ++lastSlash;

      const std::string iCachedBinary =
        cachedBinary.substr(0, lastSlash) + "i_" + cachedBinary.substr(lastSlash, chars);

      std::ofstream fs;
      fs.open(iCachedBinary.c_str());

      fs << occaCOIMain;

      fs.close();

      std::stringstream command;

      command << dev->dHandle->compiler
              << " -o " << cachedBinary
              << " -x c++"
              << ' '    << dev->dHandle->compilerFlags
              << ' '    << iCachedBinary;

      const std::string &sCommand = command.str();

      std::cout << sCommand << '\n';

      system(sCommand.c_str());
    }

    OCCA_COI_CHECK("Device: Initializing",
                   COIProcessCreateFromFile(data_.deviceID,
                                            cachedBinary.c_str(),
                                            0    , NULL,
                                            false, NULL,
                                            true , NULL,
                                            memoryAllocated ? memoryAllocated : (4 << 30), // 4 GB
                                            NULL,
                                            &(data_.chiefID)) );
{'fraction_non_alphanumeric': 0.11479131497758892, 'fraction_numerical': 0.0028232143896617964, 'mean_word_length': 1.3650192731277533, 'pattern_counts': {'":': 0, '<': 310, '<?xml version=': 0, '>': 181, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 252, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4069671', 'n_tokens_mistral': 9725, 'n_tokens_neox': 9108, 'n_words': 1824}