<commit_before>//===-- Support/FoldingSet.cpp - Uniquing Hash Set --------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements a hash set that can be used to remove duplication of
// nodes in a graph.  This code was originally created by <NAME> for use
// with SelectionDAGCSEMap, but was isolated to provide use across the llvm code
// set. 
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/FoldingSet.h"
#include "llvm/Support/MathExtras.h"
#include <cassert>
#include <cstring>
using namespace llvm;

//===----------------------------------------------------------------------===//
// FoldingSetNodeID Implementation

/// Add* - Add various data types to Bit data.
///
void FoldingSetNodeID::AddPointer(const void *Ptr) {
  // Note: this adds pointers to the hash using sizes and endianness that
  // depend on the host.  It doesn't matter however, because hashing on
  // pointer values in inherently unstable.  Nothing  should depend on the 
  // ordering of nodes in the folding set.
  intptr_t PtrI = (intptr_t)Ptr;
  Bits.push_back(unsigned(PtrI));
  if (sizeof(intptr_t) > sizeof(unsigned))
    Bits.push_back(unsigned(uint64_t(PtrI) >> 32));
}
void FoldingSetNodeID::AddInteger(signed I) {
  Bits.push_back(I);
}
void FoldingSetNodeID::AddInteger(unsigned I) {
  Bits.push_back(I);
}
void FoldingSetNodeID::AddInteger(int64_t I) {
  AddInteger((uint64_t)I);
}
void FoldingSetNodeID::AddInteger(uint64_t I) {
  Bits.push_back(unsigned(I));
  
  // If the integer is small, encode it just as 32-bits.
  if ((uint64_t)(int)I != I)
    Bits.push_back(unsigned(I >> 32));
}
void FoldingSetNodeID::AddFloat(float F) {
  Bits.push_back(FloatToBits(F));
}
void FoldingSetNodeID::AddDouble(double D) {
 AddInteger(DoubleToBits(D));
}
void FoldingSetNodeID::AddString(const std::string &String) {
  unsigned Size = static_cast<unsigned>(String.size());
  Bits.push_back(Size);
  if (!Size) return;

  unsigned Units = Size / 4;
  unsigned Pos = 0;
  const unsigned *Base = (const unsigned *)String.data();
  
  // If the string is aligned do a bulk transfer.
  if (!((intptr_t)Base & 3)) {
    Bits.append(Base, Base + Units);
    Pos = (Units + 1) * 4;
  } else {
    // Otherwise do it the hard way.
    for ( Pos += 4; Pos <= Size; Pos += 4) {
      unsigned V = ((unsigned char)String[Pos - 4] << 24) |
                   ((unsigned char)String[Pos - 3] << 16) |
                   ((unsigned char)String[Pos - 2] << 8) |
                    (unsigned char)String[Pos - 1];
      Bits.push_back(V);
    }
  }
  
  // With the leftover bits.
  unsigned V = 0;
  // Pos will have overshot size by 4 - #bytes left over. 
  switch (Pos - Size) {
  case 1: V = (V << 8) | (unsigned char)String[Size - 3]; // Fall thru.
  case 2: V = (V << 8) | (unsigned char)String[Size - 2]; // Fall thru.
  case 3: V = (V << 8) | (unsigned char)String[Size - 1]; break;
  default: return; // Nothing left.
  }

  Bits.push_back(V);
}

/// ComputeHash - Compute a strong hash value for this FoldingSetNodeID, used to 
/// lookup the node in the FoldingSetImpl.
unsigned FoldingSetNodeID::ComputeHash() const {
  // This is adapted from SuperFastHash by <NAME>.
  unsigned Hash = static_cast<unsigned>(Bits.size());
  for (const unsigned *BP = &Bits[0], *E = BP+Bits.size(); BP != E; ++BP) {
    unsigned Data = *BP;
    Hash         += Data & 0xFFFF;
    unsigned Tmp  = ((Data >> 16) << 11) ^ Hash;
    Hash          = (Hash << 16) ^ Tmp;
    Hash         += Hash >> 11;
  }
  
  // Force "avalanching" of final 127 bits.
  Hash ^= Hash << 3;
  Hash += Hash >> 5;
  Hash ^= Hash << 4;
  Hash += Hash >> 17;
  Hash ^= Hash << 25;
  Hash += Hash >> 6;
  return Hash;
}

/// operator== - Used to compare two nodes to each other.
///
bool FoldingSetNodeID::operator==(const FoldingSetNodeID &RHS)const{
  if (Bits.size() != RHS.Bits.size()) return false;
  return memcmp(&Bits[0], &RHS.Bits[0], Bits.size()*sizeof(Bits[0])) == 0;
}


//===----------------------------------------------------------------------===//
/// Helper functions for FoldingSetImpl.

/// GetNextPtr - In order to save space, each bucket is a
/// singly-linked-list. In order to make deletion more efficient, we make
/// the list circular, so we can delete a node without computing its hash.
/// The problem with this is that the start of the hash buckets are not
/// Nodes.  If NextInBucketPtr is a bucket pointer, this method returns null:
/// use GetBucketPtr when this happens.
static FoldingSetImpl::Node *GetNextPtr(void *NextInBucketPtr) {
  // The low bit is set if this is the pointer back to the bucket.
  if (reinterpret_cast<intptr_t>(NextInBucketPtr) & 1)
    return 0;
  
  return static_cast<FoldingSetImpl::Node*>(NextInBucketPtr);
}


/// testing.
static void **GetBucketPtr(void *NextInBucketPtr) {
  intptr_t Ptr = reinterpret_cast<intptr_t>(NextInBucketPtr);
  assert((Ptr & 1) && "Not a bucket pointer");
  return reinterpret_cast<void**>(Ptr & ~intptr_t(1));
}

/// GetBucketFor - Hash the specified node ID and return the hash bucket for
/// the specified ID.
static void **GetBucketFor(const FoldingSetNodeID &ID,
                           void **Buckets, unsigned NumBuckets) {
  // NumBuckets is always a power of 2.
  unsigned BucketNum = ID.ComputeHash() & (NumBuckets-1);
  return Buckets + BucketNum;
}

//===----------------------------------------------------------------------===//
// FoldingSetImpl Implementation

FoldingSetImpl::FoldingSetImpl(unsigned Log2InitSize) : NumNodes(0) {
  assert(5 < Log2InitSize && Log2InitSize < 32 &&
         "Initial hash table size out of range");
  NumBuckets = 1 << Log2InitSize;
  Buckets = new void*[NumBuckets+1];
  memset(Buckets, 0, NumBuckets*sizeof(void*));
  
  // Set the very last bucket to be a non-null "pointer".
  Buckets[NumBuckets] = reinterpret_cast<void*>(-2);
}
FoldingSetImpl::~FoldingSetImpl() {
  delete [] Buckets;
}<commit_msg>Consistently set "Buckets[NumBuckets] = reinterpret_cast<void*>(-1)" throughout FoldingSet.cpp.<commit_after>//===-- Support/FoldingSet.cpp - Uniquing Hash Set --------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements a hash set that can be used to remove duplication of
// nodes in a graph.  This code was originally created by <NAME> for use
// with SelectionDAGCSEMap, but was isolated to provide use across the llvm code
// set. 
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/FoldingSet.h"
#include "llvm/Support/MathExtras.h"
#include <cassert>
#include <cstring>
using namespace llvm;

//===----------------------------------------------------------------------===//
// FoldingSetNodeID Implementation

/// Add* - Add various data types to Bit data.
///
void FoldingSetNodeID::AddPointer(const void *Ptr) {
  // Note: this adds pointers to the hash using sizes and endianness that
  // depend on the host.  It doesn't matter however, because hashing on
  // pointer values in inherently unstable.  Nothing  should depend on the 
  // ordering of nodes in the folding set.
  intptr_t PtrI = (intptr_t)Ptr;
  Bits.push_back(unsigned(PtrI));
  if (sizeof(intptr_t) > sizeof(unsigned))
    Bits.push_back(unsigned(uint64_t(PtrI) >> 32));
}
void FoldingSetNodeID::AddInteger(signed I) {
  Bits.push_back(I);
}
void FoldingSetNodeID::AddInteger(unsigned I) {
  Bits.push_back(I);
}
void FoldingSetNodeID::AddInteger(int64_t I) {
  AddInteger((uint64_t)I);
}
void FoldingSetNodeID::AddInteger(uint64_t I) {
  Bits.push_back(unsigned(I));
  
  // If the integer is small, encode it just as 32-bits.
  if ((uint64_t)(int)I != I)
    Bits.push_back(unsigned(I >> 32));
}
void FoldingSetNodeID::AddFloat(float F) {
  Bits.push_back(FloatToBits(F));
}
void FoldingSetNodeID::AddDouble(double D) {
 AddInteger(DoubleToBits(D));
}
void FoldingSetNodeID::AddString(const std::string &String) {
  unsigned Size = static_cast<unsigned>(String.size());
  Bits.push_back(Size);
  if (!Size) return;

  unsigned Units = Size / 4;
  unsigned Pos = 0;
  const unsigned *Base = (const unsigned *)String.data();
  
  // If the string is aligned do a bulk transfer.
  if (!((intptr_t)Base & 3)) {
    Bits.append(Base, Base + Units);
    Pos = (Units + 1) * 4;
  } else {
    // Otherwise do it the hard way.
    for ( Pos += 4; Pos <= Size; Pos += 4) {
      unsigned V = ((unsigned char)String[Pos - 4] << 24) |
                   ((unsigned char)String[Pos - 3] << 16) |
                   ((unsigned char)String[Pos - 2] << 8) |
                    (unsigned char)String[Pos - 1];
      Bits.push_back(V);
    }
  }
  
  // With the leftover bits.
  unsigned V = 0;
  // Pos will have overshot size by 4 - #bytes left over. 
  switch (Pos - Size) {
  case 1: V = (V << 8) | (unsigned char)String[Size - 3]; // Fall thru.
  case 2: V = (V << 8) | (unsigned char)String[Size - 2]; // Fall thru.
  case 3: V = (V << 8) | (unsigned char)String[Size - 1]; break;
  default: return; // Nothing left.
  }

  Bits.push_back(V);
}

/// ComputeHash - Compute a strong hash value for this FoldingSetNodeID, used to 
/// lookup the node in the FoldingSetImpl.
unsigned FoldingSetNodeID::ComputeHash() const {
  // This is adapted from SuperFastHash by <NAME>.
  unsigned Hash = static_cast<unsigned>(Bits.size());
  for (const unsigned *BP = &Bits[0], *E = BP+Bits.size(); BP != E; ++BP) {
    unsigned Data = *BP;
    Hash         += Data & 0xFFFF;
    unsigned Tmp  = ((Data >> 16) << 11) ^ Hash;
    Hash          = (Hash << 16) ^ Tmp;
    Hash         += Hash >> 11;
  }
  
  // Force "avalanching" of final 127 bits.
  Hash ^= Hash << 3;
  Hash += Hash >> 5;
  Hash ^= Hash << 4;
  Hash += Hash >> 17;
  Hash ^= Hash << 25;
  Hash += Hash >> 6;
  return Hash;
}

/// operator== - Used to compare two nodes to each other.
///
bool FoldingSetNodeID::operator==(const FoldingSetNodeID &RHS)const{
  if (Bits.size() != RHS.Bits.size()) return false;
  return memcmp(&Bits[0], &RHS.Bits[0], Bits.size()*sizeof(Bits[0])) == 0;
}


//===----------------------------------------------------------------------===//
/// Helper functions for FoldingSetImpl.

/// GetNextPtr - In order to save space, each bucket is a
/// singly-linked-list. In order to make deletion more efficient, we make
/// the list circular, so we can delete a node without computing its hash.
/// The problem with this is that the start of the hash buckets are not
/// Nodes.  If NextInBucketPtr is a bucket pointer, this method returns null:
/// use GetBucketPtr when this happens.
static FoldingSetImpl::Node *GetNextPtr(void *NextInBucketPtr) {
  // The low bit is set if this is the pointer back to the bucket.
  if (reinterpret_cast<intptr_t>(NextInBucketPtr) & 1)
    return 0;
  
  return static_cast<FoldingSetImpl::Node*>(NextInBucketPtr);
}


/// testing.
static void **GetBucketPtr(void *NextInBucketPtr) {
  intptr_t Ptr = reinterpret_cast<intptr_t>(NextInBucketPtr);
  assert((Ptr & 1) && "Not a bucket pointer");
  return reinterpret_cast<void**>(Ptr & ~intptr_t(1));
}

/// GetBucketFor - Hash the specified node ID and return the hash bucket for
/// the specified ID.
static void **GetBucketFor(const FoldingSetNodeID &ID,
                           void **Buckets, unsigned NumBuckets) {
  // NumBuckets is always a power of 2.
  unsigned BucketNum = ID.ComputeHash() & (NumBuckets-1);
  return Buckets + BucketNum;
}

//===----------------------------------------------------------------------===//
// FoldingSetImpl Implementation

FoldingSetImpl::FoldingSetImpl(unsigned Log2InitSize) : NumNodes(0) {
  assert(5 < Log2InitSize && Log2InitSize < 32 &&
         "Initial hash table size out of range");
  NumBuckets = 1 << Log2InitSize;
  Buckets = new void*[NumBuckets+1];
  memset(Buckets, 0, NumBuckets*sizeof(void*));
  
  // Set the very last bucket to be a non-null "pointer".
  Buckets[NumBuckets] = reinterpret_cast<void*>(-1);
}
FoldingSetImpl::~FoldingSetImpl() {
  delete [] Buckets;
}
{'fraction_non_alphanumeric': 0.193338683788122, 'fraction_numerical': 0.013884430176565009, 'mean_word_length': 2.5420693575895394, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '486616', 'n_tokens_mistral': 4071, 'n_tokens_neox': 3843, 'n_words': 1211}