<commit_before>// ======================================================================== //
// Copyright 2009-2017 Intel Corporation                                    //
//                                                                          //
// Licensed under the Apache License, Version 2.0 (the "License");          //
// you may not use this file except in compliance with the License.         //
// You may obtain a copy of the License at                                  //
//                                                                          //
//     http://www.apache.org/licenses/LICENSE-2.0                           //
//                                                                          //
// Unless required by applicable law or agreed to in writing, software      //
// distributed under the License is distributed on an "AS IS" BASIS,        //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and      //
// limitations under the License.                                           //
// ======================================================================== //

#pragma once

#include "default.h"
#include "device.h"
#include "scene.h"
#include "primref.h"

namespace embree
{
  class FastAllocator 
  {
    /*! maximal supported alignment */
    static const size_t maxAlignment = 64;

    /*! maximal allocation size */

    /* default settings */
    //static const size_t defaultBlockSize = 4096;
#define maxAllocationSize size_t(4*1024*1024-maxAlignment)
    static const size_t MAX_THREAD_USED_BLOCK_SLOTS = 8;
    
  public:

    struct ThreadLocal2;
    enum AllocationType { ALIGNED_MALLOC, OS_MALLOC, SHARED, ANY_TYPE };

    /*! Per thread structure holding the current memory block. */
    struct __aligned(64) ThreadLocal 
    {
      ALIGNED_CLASS_(64);
    public:

      /*! Constructor for usage with ThreadLocalData */
      __forceinline ThreadLocal (ThreadLocal2* parent) 
	: parent(parent), ptr(nullptr), cur(0), end(0), allocBlockSize(0), bytesUsed(0), bytesWasted(0) {}

      /*! initialize allocator */
      void init(FastAllocator* alloc) 
      {
        ptr = nullptr;
	cur = end = 0;
	bytesWasted = 0;
        bytesUsed = 0;
        allocBlockSize = 0;
        if (alloc) allocBlockSize = alloc->defaultBlockSize;
      }

      /* Allocate aligned memory from the threads memory block. */
      __forceinline void* malloc(FastAllocator* alloc, size_t bytes, size_t align = 16) 
      {
        parent->bind(alloc);

        assert(align <= maxAlignment);
	bytesUsed += bytes;
	
        /* try to allocate in local block */
	size_t ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;
        
        /* if allocation is too large allocate with parent allocator */
        if (4*bytes > allocBlockSize) {
          return alloc->malloc(bytes,maxAlignment,false);
	}

        /* get new partial block if allocation failed */
        size_t blockSize = allocBlockSize;
        ptr = (char*) alloc->malloc(blockSize,maxAlignment,true);
 	bytesWasted += end-cur;
	cur = 0; end = blockSize;
	
        /* retry allocation */
	ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;

        /* get new full block if allocation failed */
        blockSize = allocBlockSize;
        ptr = (char*) alloc->malloc(blockSize,maxAlignment,false);
	bytesWasted += end-cur;
	cur = 0; end = blockSize;
	
        /* retry allocation */
	ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;
	
        /* should never happen as large allocations get handled specially above */
        assert(false);
        return nullptr;
      }

      /*! returns amount of used bytes */
      __forceinline size_t getUsedBytes() const { return bytesUsed; }
      
      /*! returns amount of wasted bytes */
      __forceinline size_t getWastedBytes() const { return bytesWasted + (end-cur); }

    private:
      ThreadLocal2* parent;
      char*  ptr;            //!< pointer to memory block
      size_t cur;            //!< current location of the allocator
      size_t end;            //!< end of the memory block
      size_t allocBlockSize; //!< block size for allocations
      size_t bytesUsed;      //!< number of total bytes allocated
      size_t bytesWasted;    //!< number of bytes wasted
    };

    /*! Two thread local structures. */
    struct __aligned(64) ThreadLocal2
    {
      ALIGNED_CLASS_(64);
    public:

      __forceinline ThreadLocal2()
        : alloc(nullptr), alloc0(this), alloc1(this) {}

      /*! bind to fast allocator */
      __forceinline void bind(FastAllocator* alloc_i) 
      {
        assert(alloc_i);
        if (alloc == alloc_i) return;
        Lock<SpinLock> lock(mutex);
        //if (alloc == alloc_i) return; // not required as only one thread calls bind
        if (alloc) {
          alloc->bytesUsed   += alloc0.getUsedBytes()   + alloc1.getUsedBytes();
          alloc->bytesWasted += alloc0.getWastedBytes() + alloc1.getWastedBytes();
        }
        alloc0.init(alloc_i);
        alloc1.init(alloc_i);
        alloc = alloc_i;
        alloc->join(this);
      }

      /*! unbind to fast allocator */
      void unbind(FastAllocator* alloc_i) 
      {
        assert(alloc_i);
        if (alloc != alloc_i) return;
        Lock<SpinLock> lock(mutex);
        if (alloc != alloc_i) return; // required as a different thread calls unbind
        alloc->bytesUsed   += alloc0.getUsedBytes()   + alloc1.getUsedBytes();
        alloc->bytesWasted += alloc0.getWastedBytes() + alloc1.getWastedBytes();
        alloc0.init(nullptr);
        alloc1.init(nullptr);
        alloc = nullptr;
      }

    public:
      SpinLock mutex;        //!< required as unbind is called from other threads
      FastAllocator* alloc;  //!< parent allocator
      ThreadLocal alloc0;
      ThreadLocal alloc1;
    };

    FastAllocator (Device* device, bool osAllocation) 
      : device(device), slotMask(0), usedBlocks(nullptr), freeBlocks(nullptr), use_single_mode(false), defaultBlockSize(PAGE_SIZE), 
        growSize(PAGE_SIZE), log2_grow_size_scale(0), bytesUsed(0), bytesWasted(0), atype(osAllocation ? OS_MALLOC : ALIGNED_MALLOC),
        primrefarray(device)
    {
      for (size_t i=0; i<MAX_THREAD_USED_BLOCK_SLOTS; i++)
      {
        threadUsedBlocks[i] = nullptr;
        threadBlocks[i] = nullptr;
        assert(!slotMutex[i].isLocked());
      }
    } 

    ~FastAllocator () { 
      clear();
    }<commit_msg>making alloc member atomic, required as accessed without lock from different threads<commit_after>// ======================================================================== //
// Copyright 2009-2017 Intel Corporation                                    //
//                                                                          //
// Licensed under the Apache License, Version 2.0 (the "License");          //
// you may not use this file except in compliance with the License.         //
// You may obtain a copy of the License at                                  //
//                                                                          //
//     http://www.apache.org/licenses/LICENSE-2.0                           //
//                                                                          //
// Unless required by applicable law or agreed to in writing, software      //
// distributed under the License is distributed on an "AS IS" BASIS,        //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and      //
// limitations under the License.                                           //
// ======================================================================== //

#pragma once

#include "default.h"
#include "device.h"
#include "scene.h"
#include "primref.h"

namespace embree
{
  class FastAllocator 
  {
    /*! maximal supported alignment */
    static const size_t maxAlignment = 64;

    /*! maximal allocation size */

    /* default settings */
    //static const size_t defaultBlockSize = 4096;
#define maxAllocationSize size_t(4*1024*1024-maxAlignment)
    static const size_t MAX_THREAD_USED_BLOCK_SLOTS = 8;
    
  public:

    struct ThreadLocal2;
    enum AllocationType { ALIGNED_MALLOC, OS_MALLOC, SHARED, ANY_TYPE };

    /*! Per thread structure holding the current memory block. */
    struct __aligned(64) ThreadLocal 
    {
      ALIGNED_CLASS_(64);
    public:

      /*! Constructor for usage with ThreadLocalData */
      __forceinline ThreadLocal (ThreadLocal2* parent) 
	: parent(parent), ptr(nullptr), cur(0), end(0), allocBlockSize(0), bytesUsed(0), bytesWasted(0) {}

      /*! initialize allocator */
      void init(FastAllocator* alloc) 
      {
        ptr = nullptr;
	cur = end = 0;
	bytesWasted = 0;
        bytesUsed = 0;
        allocBlockSize = 0;
        if (alloc) allocBlockSize = alloc->defaultBlockSize;
      }

      /* Allocate aligned memory from the threads memory block. */
      __forceinline void* malloc(FastAllocator* alloc, size_t bytes, size_t align = 16) 
      {
        parent->bind(alloc);

        assert(align <= maxAlignment);
	bytesUsed += bytes;
	
        /* try to allocate in local block */
	size_t ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;
        
        /* if allocation is too large allocate with parent allocator */
        if (4*bytes > allocBlockSize) {
          return alloc->malloc(bytes,maxAlignment,false);
	}

        /* get new partial block if allocation failed */
        size_t blockSize = allocBlockSize;
        ptr = (char*) alloc->malloc(blockSize,maxAlignment,true);
 	bytesWasted += end-cur;
	cur = 0; end = blockSize;
	
        /* retry allocation */
	ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;

        /* get new full block if allocation failed */
        blockSize = allocBlockSize;
        ptr = (char*) alloc->malloc(blockSize,maxAlignment,false);
	bytesWasted += end-cur;
	cur = 0; end = blockSize;
	
        /* retry allocation */
	ofs = (align - cur) & (align-1); 
        cur += bytes + ofs;
        if (likely(cur <= end)) { bytesWasted += ofs; return &ptr[cur - bytes]; }
	cur -= bytes + ofs;
	
        /* should never happen as large allocations get handled specially above */
        assert(false);
        return nullptr;
      }

      /*! returns amount of used bytes */
      __forceinline size_t getUsedBytes() const { return bytesUsed; }
      
      /*! returns amount of wasted bytes */
      __forceinline size_t getWastedBytes() const { return bytesWasted + (end-cur); }

    private:
      ThreadLocal2* parent;
      char*  ptr;            //!< pointer to memory block
      size_t cur;            //!< current location of the allocator
      size_t end;            //!< end of the memory block
      size_t allocBlockSize; //!< block size for allocations
      size_t bytesUsed;      //!< number of total bytes allocated
      size_t bytesWasted;    //!< number of bytes wasted
    };

    /*! Two thread local structures. */
    struct __aligned(64) ThreadLocal2
    {
      ALIGNED_CLASS_(64);
    public:

      __forceinline ThreadLocal2()
        : alloc(nullptr), alloc0(this), alloc1(this) {}

      /*! bind to fast allocator */
      __forceinline void bind(FastAllocator* alloc_i) 
      {
        assert(alloc_i);
        if (alloc.load() == alloc_i) return;
        Lock<SpinLock> lock(mutex);
        //if (alloc.load() == alloc_i) return; // not required as only one thread calls bind
        if (alloc.load()) {
          alloc.load()->bytesUsed   += alloc0.getUsedBytes()   + alloc1.getUsedBytes();
          alloc.load()->bytesWasted += alloc0.getWastedBytes() + alloc1.getWastedBytes();
        }
        alloc0.init(alloc_i);
        alloc1.init(alloc_i);
        alloc.store(alloc_i);
        alloc_i->join(this);
      }

      /*! unbind to fast allocator */
      void unbind(FastAllocator* alloc_i) 
      {
        assert(alloc_i);
        if (alloc.load() != alloc_i) return;
        Lock<SpinLock> lock(mutex);
        if (alloc.load() != alloc_i) return; // required as a different thread calls unbind
        alloc.load()->bytesUsed   += alloc0.getUsedBytes()   + alloc1.getUsedBytes();
        alloc.load()->bytesWasted += alloc0.getWastedBytes() + alloc1.getWastedBytes();
        alloc0.init(nullptr);
        alloc1.init(nullptr);
        alloc.store(nullptr);
      }

    public:
      SpinLock mutex;        //!< required as unbind is called from other threads
      std::atomic<FastAllocator*> alloc;  //!< parent allocator
      ThreadLocal alloc0;
      ThreadLocal alloc1;
    };

    FastAllocator (Device* device, bool osAllocation) 
      : device(device), slotMask(0), usedBlocks(nullptr), freeBlocks(nullptr), use_single_mode(false), defaultBlockSize(PAGE_SIZE), 
        growSize(PAGE_SIZE), log2_grow_size_scale(0), bytesUsed(0), bytesWasted(0), atype(osAllocation ? OS_MALLOC : ALIGNED_MALLOC),
        primrefarray(device)
    {
      for (size_t i=0; i<MAX_THREAD_USED_BLOCK_SLOTS; i++)
      {
        threadUsedBlocks[i] = nullptr;
        threadBlocks[i] = nullptr;
        assert(!slotMutex[i].isLocked());
      }
    } 

    ~FastAllocator () { 
      clear();
    }
{'fraction_non_alphanumeric': 0.1312919766014299, 'fraction_numerical': 0.01155484942586842, 'mean_word_length': 2.0890028998438543, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5358320', 'n_tokens_mistral': 3907, 'n_tokens_neox': 3732, 'n_words': 1081}