<commit_before>//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the section-based memory manager used by the MCJIT
// execution engine and RuntimeDyld
//
//===----------------------------------------------------------------------===//

#include "llvm/Config/config.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Process.h"

namespace llvm {

uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
                                                   unsigned Alignment,
                                                   unsigned SectionID,
                                                   StringRef SectionName,
                                                   bool IsReadOnly) {
  if (IsReadOnly)
    return allocateSection(RODataMem, Size, Alignment);
  return allocateSection(RWDataMem, Size, Alignment);
}

uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
                                                   unsigned Alignment,
                                                   unsigned SectionID,
                                                   StringRef SectionName) {
  return allocateSection(CodeMem, Size, Alignment);
}

uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup,
                                               uintptr_t Size,
                                               unsigned Alignment) {
  if (!Alignment)
    Alignment = 16;

  assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");

  uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1)/Alignment + 1);
  uintptr_t Addr = 0;

  // Look in the list of free memory regions and use a block there if one
  // is available.
  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
    if (FreeMB.Free.size() >= RequiredSize) {
      Addr = (uintptr_t)FreeMB.Free.base();
      uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
      // Align the address.
      Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);

      if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
        // The part of the block we're giving out to the user is now pending
        MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));

        // Remember this pending block, such that future allocations can just
        // modify it rather than creating a new one
        FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
      } else {
        sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
        PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
      }

      // Remember how much free space is now left in this block
      FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
      return (uint8_t*)Addr;
    }
  }

  // No pre-allocated free block was large enough. Allocate a new memory region.
  // Note that all sections get allocated as read-write.  The permissions will
  // be updated later based on memory group.
  //
  // FIXME: It would be useful to define a default allocation size (or add
  // it as a constructor parameter) to minimize the number of allocations.
  //
  // FIXME: Initialize the Near member for each memory group to avoid
  // interleaving.
  // Note: this would not be sufficient as the Near member is just a hint
  // to mmap.
  std::error_code ec;
  sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(RequiredSize,
                                                          &MemGroup.Near,
                                                          sys::Memory::MF_READ |
                                                            sys::Memory::MF_WRITE,
                                                          ec);
  if (ec) {
    // FIXME: Add error propagation to the interface.
    return nullptr;
  }

  // To be sure that there is no interleaving, enforce the monotonic decrease
  // of the memory adddress.  This is required by libgcc's _Unwind_Find_FDE
  // which does a linear search (of the top level dwarf objects) and note in
  // the code:
  // "Note that pc_begin is sorted descending, and we expect objects to be
  // non-overlapping. "
  using CheckerType = bool (*)(void *, void *);
  CheckerType isNotMonotonicLinux = [](void *newAlloc, void *target) {
    return (size_t)newAlloc > (size_t)target;
  };
  CheckerType isNotMonotonicAlternateOrder = [](void *newAlloc, void *target) {
    return (size_t)newAlloc < (size_t)target;
  };
  auto getMonotonicChecker = [=]() {
    std::error_code ec;
    sys::MemoryBlock MB2 = sys::Memory::allocateMappedMemory(RequiredSize,
                                                             &MemGroup.Near,
                                                             sys::Memory::MF_READ |
                                                             sys::Memory::MF_WRITE,
                                                             ec);
    bool linuxType = false;
    if ((size_t)MB2.base() < (size_t)MB.base()) {
      linuxType = true;
    }
    sys::Memory::releaseMappedMemory(MB2);
    if (linuxType) {
      return isNotMonotonicLinux;
    } else {
      return isNotMonotonicAlternateOrder;
    }
  };
  static CheckerType isNotMonotonic = getMonotonicChecker();

  if (MemGroup.Near.base() && isNotMonotonic(MB.base(), MemGroup.Near.base())) {
    SmallVector<sys::MemoryBlock, 16> MemToDelete;
    while (isNotMonotonic(MB.base(), MemGroup.Near.base())) {

      // Hold on to the memory until we get an acceptable one
      // just so that we are not given it back.
      MemToDelete.push_back(MB);
      MB = sys::Memory::allocateMappedMemory(RequiredSize,
                                             &MemGroup.Near,
                                             sys::Memory::MF_READ |
                                               sys::Memory::MF_WRITE,
                                             ec);
      if (ec) {
        for (sys::MemoryBlock &Block : MemToDelete) {
          sys::Memory::releaseMappedMemory(Block);
        }
        // FIXME: Add error propagation to the interface.
        return nullptr;
      }
    }
    for (sys::MemoryBlock &Block : MemToDelete) {
      sys::Memory::releaseMappedMemory(Block);
    }
  }

  // Save this address as the basis for our next request
  MemGroup.Near = MB;

  // Remember that we allocated this memory
  MemGroup.AllocatedMem.push_back(MB);
  Addr = (uintptr_t)MB.base();
  uintptr_t EndOfBlock = Addr + MB.size();

  // Align the address.
  Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);

  // The part of the block we're giving out to the user is now pending
  MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));

  // The allocateMappedMemory may allocate much more memory than we need. In
  // this case, we store the unused memory as a free memory block.<commit_msg>Give up after too many iteration<commit_after>//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the section-based memory manager used by the MCJIT
// execution engine and RuntimeDyld
//
//===----------------------------------------------------------------------===//

#include "llvm/Config/config.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Process.h"

namespace llvm {

uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
                                                   unsigned Alignment,
                                                   unsigned SectionID,
                                                   StringRef SectionName,
                                                   bool IsReadOnly) {
  if (IsReadOnly)
    return allocateSection(RODataMem, Size, Alignment);
  return allocateSection(RWDataMem, Size, Alignment);
}

uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
                                                   unsigned Alignment,
                                                   unsigned SectionID,
                                                   StringRef SectionName) {
  return allocateSection(CodeMem, Size, Alignment);
}

uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup,
                                               uintptr_t Size,
                                               unsigned Alignment) {
  if (!Alignment)
    Alignment = 16;

  assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");

  uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1)/Alignment + 1);
  uintptr_t Addr = 0;

  // Look in the list of free memory regions and use a block there if one
  // is available.
  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
    if (FreeMB.Free.size() >= RequiredSize) {
      Addr = (uintptr_t)FreeMB.Free.base();
      uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
      // Align the address.
      Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);

      if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
        // The part of the block we're giving out to the user is now pending
        MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));

        // Remember this pending block, such that future allocations can just
        // modify it rather than creating a new one
        FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
      } else {
        sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
        PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
      }

      // Remember how much free space is now left in this block
      FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
      return (uint8_t*)Addr;
    }
  }

  // No pre-allocated free block was large enough. Allocate a new memory region.
  // Note that all sections get allocated as read-write.  The permissions will
  // be updated later based on memory group.
  //
  // FIXME: It would be useful to define a default allocation size (or add
  // it as a constructor parameter) to minimize the number of allocations.
  //
  // FIXME: Initialize the Near member for each memory group to avoid
  // interleaving.
  // Note: this would not be sufficient as the Near member is just a hint
  // to mmap.
  std::error_code ec;
  sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(RequiredSize,
                                                          &MemGroup.Near,
                                                          sys::Memory::MF_READ |
                                                            sys::Memory::MF_WRITE,
                                                          ec);
  if (ec) {
    // FIXME: Add error propagation to the interface.
    return nullptr;
  }

  // To be sure that there is no interleaving, enforce the monotonic decrease
  // of the memory adddress.  This is required by libgcc's _Unwind_Find_FDE
  // which does a linear search (of the top level dwarf objects) and note in
  // the code:
  // "Note that pc_begin is sorted descending, and we expect objects to be
  // non-overlapping. "
  using CheckerType = bool (*)(void *, void *);
  CheckerType isNotMonotonicLinux = [](void *newAlloc, void *target) {
    return (size_t)newAlloc > (size_t)target;
  };
  CheckerType isNotMonotonicAlternateOrder = [](void *newAlloc, void *target) {
    return (size_t)newAlloc < (size_t)target;
  };
  auto getMonotonicChecker = [=]() {
    std::error_code ec;
    sys::MemoryBlock MB2 = sys::Memory::allocateMappedMemory(RequiredSize,
                                                             &MemGroup.Near,
                                                             sys::Memory::MF_READ |
                                                             sys::Memory::MF_WRITE,
                                                             ec);
    bool linuxType = false;
    if ((size_t)MB2.base() < (size_t)MB.base()) {
      linuxType = true;
    }
    sys::Memory::releaseMappedMemory(MB2);
    if (linuxType) {
      return isNotMonotonicLinux;
    } else {
      return isNotMonotonicAlternateOrder;
    }
  };
  static CheckerType isNotMonotonic = getMonotonicChecker();

  int count = 0;

  if (MemGroup.Near.base() && isNotMonotonic(MB.base(), MemGroup.Near.base())) {
    SmallVector<sys::MemoryBlock, 16> MemToDelete;
    while (isNotMonotonic(MB.base(), MemGroup.Near.base())) {

      if (count > 1000) {
         // We make 1000 allocations without getting in good shape, let's
         // give up and return the first one we allocated.
         MemToDelete.push_back(MB);
         MB = MemToDelete.front();
         MemToDelete.erase(MemToDelete.begin());
         break;
      }

      // Hold on to the memory until we get an acceptable one
      // just so that we are not given it back.
      MemToDelete.push_back(MB);
      MB = sys::Memory::allocateMappedMemory(RequiredSize,
                                             &MemGroup.Near,
                                             sys::Memory::MF_READ |
                                               sys::Memory::MF_WRITE,
                                             ec);
      if (ec) {
        for (sys::MemoryBlock &Block : MemToDelete) {
          sys::Memory::releaseMappedMemory(Block);
        }
        // FIXME: Add error propagation to the interface.
        return nullptr;
      }
      ++count;
    }
    for (sys::MemoryBlock &Block : MemToDelete) {
      sys::Memory::releaseMappedMemory(Block);
    }
  }

  // Save this address as the basis for our next request
  MemGroup.Near = MB;

  // Remember that we allocated this memory
  MemGroup.AllocatedMem.push_back(MB);
  Addr = (uintptr_t)MB.base();
  uintptr_t EndOfBlock = Addr + MB.size();

  // Align the address.
  Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);

  // The part of the block we're giving out to the user is now pending
  MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));

  // The allocateMappedMemory may allocate much more memory than we need. In
  // this case, we store the unused memory as a free memory block.
{'fraction_non_alphanumeric': 0.11574261432618373, 'fraction_numerical': 0.0034399028733306356, 'mean_word_length': 1.765715351613505, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 84, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2142926', 'n_tokens_mistral': 3869, 'n_tokens_neox': 3758, 'n_words': 1165}