<commit_before>//===--- SourceManager.cpp - Track and cache source files -----------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements the SourceManager interface.
//
//===----------------------------------------------------------------------===//

#include "clang/Basic/SourceManager.h"
#include "clang/Basic/FileManager.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/System/Path.h"
#include "llvm/Bitcode/Serialize.h"
#include "llvm/Bitcode/Deserialize.h"
#include "llvm/Support/Streams.h"
#include <algorithm>
using namespace clang;
using namespace SrcMgr;
using llvm::MemoryBuffer;

//===----------------------------------------------------------------------===//
// SourceManager Helper Classes
//===----------------------------------------------------------------------===//

ContentCache::~ContentCache() {
  delete Buffer;
}

/// getSizeBytesMapped - Returns the number of bytes actually mapped for
///  this ContentCache.  This can be 0 if the MemBuffer was not actually
///  instantiated.
unsigned ContentCache::getSizeBytesMapped() const {
  return Buffer ? Buffer->getBufferSize() : 0;
}

/// getSize - Returns the size of the content encapsulated by this ContentCache.
///  This can be the size of the source file or the size of an arbitrary
///  scratch buffer.  If the ContentCache encapsulates a source file, that
///  file is not lazily brought in from disk to satisfy this query.
unsigned ContentCache::getSize() const {
  return Entry ? Entry->getSize() : Buffer->getBufferSize();
}

const llvm::MemoryBuffer *ContentCache::getBuffer() const {  
  // Lazily create the Buffer for ContentCaches that wrap files.
  if (!Buffer && Entry) {
    // FIXME: Should we support a way to not have to do this check over
    //   and over if we cannot open the file?
    Buffer = MemoryBuffer::getFile(Entry->getName(), 0, Entry->getSize());
  }
  return Buffer;
}

//===----------------------------------------------------------------------===//
// Line Table Implementation
//===----------------------------------------------------------------------===//

namespace clang {
struct LineEntry {
  /// FileOffset - The offset in this file that the line entry occurs at.
  unsigned FileOffset;
  /// LineNo - The presumed line number of this line entry: #line 4.
  unsigned LineNo;
  /// FilenameID - The ID of the filename identified by this line entry:
  /// #line 4 "foo.c".  This is -1 if not specified.
  int FilenameID;
  
  static LineEntry get(unsigned Offs, unsigned Line, int Filename) {
    LineEntry E;
    E.FileOffset = Offs;
    E.LineNo = Line;
    E.FilenameID = Filename;
    return E;
  }
};
  
  
/// LineTableInfo - This class is used to hold and unique data used to
/// represent #line information.
class LineTableInfo {
  /// FilenameIDs - This map is used to assign unique IDs to filenames in
  /// #line directives.  This allows us to unique the filenames that
  /// frequently reoccur and reference them with indices.  FilenameIDs holds
  /// the mapping from string -> ID, and FilenamesByID holds the mapping of ID
  /// to string.
  llvm::StringMap<unsigned, llvm::BumpPtrAllocator> FilenameIDs;
  std::vector<llvm::StringMapEntry<unsigned>*> FilenamesByID;
  
  /// LineEntries - This is a map from FileIDs to a list of line entries (sorted
  /// by the offset they occur in the file.
  std::map<unsigned, std::vector<LineEntry> > LineEntries;
public:
  LineTableInfo() {
  }
  
  void clear() {
    FilenameIDs.clear();
    FilenamesByID.clear();
  }
  
  ~LineTableInfo() {}
  
  unsigned getLineTableFilenameID(const char *Ptr, unsigned Len);
  const char *getFilename(unsigned ID) const {
    assert(ID < FilenamesByID.size() && "Invalid FilenameID");
    return FilenamesByID[ID]->getKeyData();
  }
  
  void AddLineNote(unsigned FID, unsigned Offset,
                   unsigned LineNo, int FilenameID);
  
  /// FindNearestLineEntry - Find the line entry nearest to FID that is before
  /// it.  If there is no line entry before Offset in FID, return null.
  const LineEntry *FindNearestLineEntry(unsigned FID, unsigned Offset);
};
} // namespace clang

unsigned LineTableInfo::getLineTableFilenameID(const char *Ptr, unsigned Len) {
  // Look up the filename in the string table, returning the pre-existing value
  // if it exists.
  llvm::StringMapEntry<unsigned> &Entry = 
    FilenameIDs.GetOrCreateValue(Ptr, Ptr+Len, ~0U);
  if (Entry.getValue() != ~0U)
    return Entry.getValue();
  
  // Otherwise, assign this the next available ID.
  Entry.setValue(FilenamesByID.size());
  FilenamesByID.push_back(&Entry);
  return FilenamesByID.size()-1;
}

/// AddLineNote - Add a line note to the line table that indicates that there
/// is a #line at the specified FID/Offset location which changes the presumed
/// location to LineNo/FilenameID.
void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
                                unsigned LineNo, int FilenameID) {
  std::vector<LineEntry> &Entries = LineEntries[FID];
  
  assert((Entries.empty() || Entries.back().FileOffset < Offset) &&
         "Adding line entries out of order!");
  
  // If this is a '#line 4' after '#line 42 "foo.h"', make sure to remember that
  // we are still in "foo.h".
  if (FilenameID == -1 && !Entries.empty())
    FilenameID = Entries.back().FilenameID;
  
  Entries.push_back(LineEntry::get(Offset, LineNo, FilenameID));
}

/// FindNearestLineEntry - Find the line entry nearest to FID that is before
/// it.  If there is no line entry before Offset in FID, return null.
const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID, 
                                                     unsigned Offset) {
  const std::vector<LineEntry> &Entries = LineEntries[FID];
  assert(!Entries.empty() && "No #line entries for this FID after all!");


  // FIXME: Dumb linear search.
  // Find the maximal element that is still before Offset.
  for (std::vector<LineEntry>::const_reverse_iterator I = Entries.rbegin(),
       E = Entries.rend(); I != E; ++I)
    if (I->FileOffset <= Offset) return &*I;
  return 0;
}


/// getLineTableFilenameID - Return the uniqued ID for the specified filename.
/// 
unsigned SourceManager::getLineTableFilenameID(const char *Ptr, unsigned Len) {
  if (LineTable == 0)
    LineTable = new LineTableInfo();
  return LineTable->getLineTableFilenameID(Ptr, Len);
}
<commit_msg>replace gimpy linear search with svelte binary search ;-)<commit_after>//===--- SourceManager.cpp - Track and cache source files -----------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements the SourceManager interface.
//
//===----------------------------------------------------------------------===//

#include "clang/Basic/SourceManager.h"
#include "clang/Basic/FileManager.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/System/Path.h"
#include "llvm/Bitcode/Serialize.h"
#include "llvm/Bitcode/Deserialize.h"
#include "llvm/Support/Streams.h"
#include <algorithm>
using namespace clang;
using namespace SrcMgr;
using llvm::MemoryBuffer;

//===----------------------------------------------------------------------===//
// SourceManager Helper Classes
//===----------------------------------------------------------------------===//

ContentCache::~ContentCache() {
  delete Buffer;
}

/// getSizeBytesMapped - Returns the number of bytes actually mapped for
///  this ContentCache.  This can be 0 if the MemBuffer was not actually
///  instantiated.
unsigned ContentCache::getSizeBytesMapped() const {
  return Buffer ? Buffer->getBufferSize() : 0;
}

/// getSize - Returns the size of the content encapsulated by this ContentCache.
///  This can be the size of the source file or the size of an arbitrary
///  scratch buffer.  If the ContentCache encapsulates a source file, that
///  file is not lazily brought in from disk to satisfy this query.
unsigned ContentCache::getSize() const {
  return Entry ? Entry->getSize() : Buffer->getBufferSize();
}

const llvm::MemoryBuffer *ContentCache::getBuffer() const {  
  // Lazily create the Buffer for ContentCaches that wrap files.
  if (!Buffer && Entry) {
    // FIXME: Should we support a way to not have to do this check over
    //   and over if we cannot open the file?
    Buffer = MemoryBuffer::getFile(Entry->getName(), 0, Entry->getSize());
  }
  return Buffer;
}

//===----------------------------------------------------------------------===//
// Line Table Implementation
//===----------------------------------------------------------------------===//

namespace clang {
struct LineEntry {
  /// FileOffset - The offset in this file that the line entry occurs at.
  unsigned FileOffset;
  /// LineNo - The presumed line number of this line entry: #line 4.
  unsigned LineNo;
  /// FilenameID - The ID of the filename identified by this line entry:
  /// #line 4 "foo.c".  This is -1 if not specified.
  int FilenameID;
  
  static LineEntry get(unsigned Offs, unsigned Line, int Filename) {
    LineEntry E;
    E.FileOffset = Offs;
    E.LineNo = Line;
    E.FilenameID = Filename;
    return E;
  }
};

inline bool operator<(const LineEntry &E, unsigned Offset) {
  return E.FileOffset < Offset;
}

inline bool operator<(unsigned Offset, const LineEntry &E) {
  return Offset < E.FileOffset;
}
  
/// LineTableInfo - This class is used to hold and unique data used to
/// represent #line information.
class LineTableInfo {
  /// FilenameIDs - This map is used to assign unique IDs to filenames in
  /// #line directives.  This allows us to unique the filenames that
  /// frequently reoccur and reference them with indices.  FilenameIDs holds
  /// the mapping from string -> ID, and FilenamesByID holds the mapping of ID
  /// to string.
  llvm::StringMap<unsigned, llvm::BumpPtrAllocator> FilenameIDs;
  std::vector<llvm::StringMapEntry<unsigned>*> FilenamesByID;
  
  /// LineEntries - This is a map from FileIDs to a list of line entries (sorted
  /// by the offset they occur in the file.
  std::map<unsigned, std::vector<LineEntry> > LineEntries;
public:
  LineTableInfo() {
  }
  
  void clear() {
    FilenameIDs.clear();
    FilenamesByID.clear();
  }
  
  ~LineTableInfo() {}
  
  unsigned getLineTableFilenameID(const char *Ptr, unsigned Len);
  const char *getFilename(unsigned ID) const {
    assert(ID < FilenamesByID.size() && "Invalid FilenameID");
    return FilenamesByID[ID]->getKeyData();
  }
  
  void AddLineNote(unsigned FID, unsigned Offset,
                   unsigned LineNo, int FilenameID);
  
  /// FindNearestLineEntry - Find the line entry nearest to FID that is before
  /// it.  If there is no line entry before Offset in FID, return null.
  const LineEntry *FindNearestLineEntry(unsigned FID, unsigned Offset);
};
} // namespace clang

unsigned LineTableInfo::getLineTableFilenameID(const char *Ptr, unsigned Len) {
  // Look up the filename in the string table, returning the pre-existing value
  // if it exists.
  llvm::StringMapEntry<unsigned> &Entry = 
    FilenameIDs.GetOrCreateValue(Ptr, Ptr+Len, ~0U);
  if (Entry.getValue() != ~0U)
    return Entry.getValue();
  
  // Otherwise, assign this the next available ID.
  Entry.setValue(FilenamesByID.size());
  FilenamesByID.push_back(&Entry);
  return FilenamesByID.size()-1;
}

/// AddLineNote - Add a line note to the line table that indicates that there
/// is a #line at the specified FID/Offset location which changes the presumed
/// location to LineNo/FilenameID.
void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset,
                                unsigned LineNo, int FilenameID) {
  std::vector<LineEntry> &Entries = LineEntries[FID];
  
  assert((Entries.empty() || Entries.back().FileOffset < Offset) &&
         "Adding line entries out of order!");
  
  // If this is a '#line 4' after '#line 42 "foo.h"', make sure to remember that
  // we are still in "foo.h".
  if (FilenameID == -1 && !Entries.empty())
    FilenameID = Entries.back().FilenameID;
  
  Entries.push_back(LineEntry::get(Offset, LineNo, FilenameID));
}

/// FindNearestLineEntry - Find the line entry nearest to FID that is before
/// it.  If there is no line entry before Offset in FID, return null.
const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID, 
                                                     unsigned Offset) {
  const std::vector<LineEntry> &Entries = LineEntries[FID];
  assert(!Entries.empty() && "No #line entries for this FID after all!");

  // It is very common for the query to be after the last #line, check this
  // first.
  if (Entries.back().FileOffset <= Offset)
    return &Entries.back();

  // Do a binary search to find the maximal element that is still before Offset.
  std::vector<LineEntry>::const_iterator I =
    std::upper_bound(Entries.begin(), Entries.end(), Offset);
  if (I == Entries.begin()) return 0;
  return &*--I;
}


/// getLineTableFilenameID - Return the uniqued ID for the specified filename.
/// 
unsigned SourceManager::getLineTableFilenameID(const char *Ptr, unsigned Len) {
  if (LineTable == 0)
    LineTable = new LineTableInfo();
  return LineTable->getLineTableFilenameID(Ptr, Len);
}

{'fraction_non_alphanumeric': 0.17462992818408324, 'fraction_numerical': 0.0021984464311886266, 'mean_word_length': 2.5566849100860045, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802411', 'n_tokens_mistral': 3810, 'n_tokens_neox': 3735, 'n_words': 1328}