<commit_before>//===--- UnicodeCharSets.h - Contains important sets of characters --------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#ifndef CLANG_LEX_UNICODECHARSETS_H
#define CLANG_LEX_UNICODECHARSETS_H

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/MutexGuard.h"
#include "llvm/Support/raw_ostream.h"

namespace {
  struct UnicodeCharRange {
    uint32_t Lower;
    uint32_t Upper;
  };
  typedef llvm::ArrayRef<UnicodeCharRange> UnicodeCharSet;

  typedef llvm::SmallPtrSet<const UnicodeCharRange *, 16> ValidatedCharSetsTy;
}

static inline ValidatedCharSetsTy &getValidatedCharSets() {
  static ValidatedCharSetsTy Validated;
  return Validated;
}

/// Returns true if each of the ranges in \p CharSet is a proper closed range
/// [min, max], and if the ranges themselves are ordered and non-overlapping.
static inline bool isValidCharSet(UnicodeCharSet CharSet) {
#ifndef NDEBUG
  static llvm::sys::Mutex ValidationMutex;

  // Check the validation cache.
  {
    llvm::MutexGuard Guard(ValidationMutex);
    if (getValidatedCharSets().count(CharSet.data()))
      return true;
  }

  // Walk through the ranges.
  uint32_t Prev = 0;
  for (UnicodeCharSet::iterator I = CharSet.begin(), E = CharSet.end();
       I != E; ++I) {
    if (Prev >= I->Lower) {
      DEBUG(llvm::dbgs() << "Upper bound 0x");
      DEBUG(llvm::dbgs().write_hex(Prev));
      DEBUG(llvm::dbgs() << " should be less than succeeding lower bound 0x");
      DEBUG(llvm::dbgs().write_hex(I->Lower) << "\n");
      return false;
    }
    if (I->Upper < I->Lower) {
      DEBUG(llvm::dbgs() << "Upper bound 0x");
      DEBUG(llvm::dbgs().write_hex(I->Lower));
      DEBUG(llvm::dbgs() << " should not be less than lower bound 0x");
      DEBUG(llvm::dbgs().write_hex(I->Upper) << "\n");
      return false;
    }
    Prev = I->Upper;
  }

  // Update the validation cache.
  {
    llvm::MutexGuard Guard(ValidationMutex);
    getValidatedCharSets().insert(CharSet.data());
  }
#endif
  return true;
}

/// Returns true if the Unicode code point \p C is within the set of
/// characters specified by \p CharSet.
LLVM_READONLY static inline bool isCharInSet(uint32_t C,
                                             UnicodeCharSet CharSet) {
  assert(isValidCharSet(CharSet));

  size_t LowPoint = 0;
  size_t HighPoint = CharSet.size();

  // Binary search the set of char ranges.
  while (HighPoint != LowPoint) {
    size_t MidPoint = (HighPoint + LowPoint) / 2;
    if (C < CharSet[MidPoint].Lower)
      HighPoint = MidPoint;
    else if (C > CharSet[MidPoint].Upper)
      LowPoint = MidPoint + 1;
    else
      return true;
  }

  return false;
}


// C11 D.1, C++11 [charname.allowed]
static const UnicodeCharRange C11AllowedIDChars[] = {
  // 1
  { 0x00A8, 0x00A8 }, { 0x00AA, 0x00AA }, { 0x00AD, 0x00AD },
  { 0x00AF, 0x00AF }, { 0x00B2, 0x00B5 }, { 0x00B7, 0x00BA },
  { 0x00BC, 0x00BE }, { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 },
  { 0x00F8, 0x00FF },
  // 2
  { 0x0100, 0x167F }, { 0x1681, 0x180D }, { 0x180F, 0x1FFF },
  // 3
  { 0x200B, 0x200D }, { 0x202A, 0x202E }, { 0x203F, 0x2040 },
  { 0x2054, 0x2054 }, { 0x2060, 0x206F },
  // 4
  { 0x2070, 0x218F }, { 0x2460, 0x24FF }, { 0x2776, 0x2793 },
  { 0x2C00, 0x2DFF }, { 0x2E80, 0x2FFF },
  // 5
  { 0x3004, 0x3007 }, { 0x3021, 0x302F }, { 0x3031, 0x303F },
  // 6
  { 0x3040, 0xD7FF },
  // 7<commit_msg>Use isCharInSet from llvm/Support/UnicodeCharRanges.h, added a test for double-width characters in FixIt-hints.<commit_after>//===--- UnicodeCharSets.h - Contains important sets of characters --------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#ifndef CLANG_LEX_UNICODECHARSETS_H
#define CLANG_LEX_UNICODECHARSETS_H

#include "llvm/Support/UnicodeCharRanges.h"

// C11 D.1, C++11 [charname.allowed]
static const UnicodeCharRange C11AllowedIDChars[] = {
  // 1
  { 0x00A8, 0x00A8 }, { 0x00AA, 0x00AA }, { 0x00AD, 0x00AD },
  { 0x00AF, 0x00AF }, { 0x00B2, 0x00B5 }, { 0x00B7, 0x00BA },
  { 0x00BC, 0x00BE }, { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 },
  { 0x00F8, 0x00FF },
  // 2
  { 0x0100, 0x167F }, { 0x1681, 0x180D }, { 0x180F, 0x1FFF },
  // 3
  { 0x200B, 0x200D }, { 0x202A, 0x202E }, { 0x203F, 0x2040 },
  { 0x2054, 0x2054 }, { 0x2060, 0x206F },
  // 4
  { 0x2070, 0x218F }, { 0x2460, 0x24FF }, { 0x2776, 0x2793 },
  { 0x2C00, 0x2DFF }, { 0x2E80, 0x2FFF },
  // 5
  { 0x3004, 0x3007 }, { 0x3021, 0x302F }, { 0x3031, 0x303F },
  // 6
  { 0x3040, 0xD7FF },
  // 7
{'fraction_non_alphanumeric': 0.17440929114937925, 'fraction_numerical': 0.09411293552262716, 'mean_word_length': 2.7276119402985075, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1649930', 'n_tokens_mistral': 2193, 'n_tokens_neox': 1871, 'n_words': 432}