<commit_before>//===- LLLexer.cpp - Lexer for .ll Files ----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implement the Lexer for .ll files.
//
//===----------------------------------------------------------------------===//

#include "LLLexer.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Instruction.h"
#include "llvm/LLVMContext.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Assembly/Parser.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/raw_ostream.h"
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <cstring>
using namespace llvm;

bool LLLexer::Error(LocTy ErrorLoc, const Twine &Msg) const {
  ErrorInfo = SM.GetMessage(ErrorLoc, SourceMgr::DK_Error, Msg);
  return true;
}

//===----------------------------------------------------------------------===//
// Helper functions.
//===----------------------------------------------------------------------===//

// atoull - Convert an ascii string of decimal digits into the unsigned long
// long representation... this does not have to do input error checking,
// because we know that the input will be matched by a suitable regex...
//
uint64_t LLLexer::atoull(const char *Buffer, const char *End) {
  uint64_t Result = 0;
  for (; Buffer != End; Buffer++) {
    uint64_t OldRes = Result;
    Result *= 10;
    Result += *Buffer-'0';
    if (Result < OldRes) {  // Uh, oh, overflow detected!!!
      Error("constant bigger than 64 bits detected!");
      return 0;
    }
  }
  return Result;
}

uint64_t LLLexer::HexIntToVal(const char *Buffer, const char *End) {
  uint64_t Result = 0;
  for (; Buffer != End; ++Buffer) {
    uint64_t OldRes = Result;
    Result *= 16;
    char C = *Buffer;
    if (C >= '0' && C <= '9')
      Result += C-'0';
    else if (C >= 'A' && C <= 'F')
      Result += C-'A'+10;
    else if (C >= 'a' && C <= 'f')
      Result += C-'a'+10;

    if (Result < OldRes) {   // Uh, oh, overflow detected!!!
      Error("constant bigger than 64 bits detected!");
      return 0;
    }
  }
  return Result;
}

void LLLexer::HexToIntPair(const char *Buffer, const char *End,
                           uint64_t Pair[2]) {
  Pair[0] = 0;
  for (int i=0; i<16; i++, Buffer++) {
    assert(Buffer != End);
    Pair[0] *= 16;
    char C = *Buffer;
    if (C >= '0' && C <= '9')
      Pair[0] += C-'0';
    else if (C >= 'A' && C <= 'F')
      Pair[0] += C-'A'+10;
    else if (C >= 'a' && C <= 'f')
      Pair[0] += C-'a'+10;
  }
  Pair[1] = 0;
  for (int i=0; i<16 && Buffer != End; i++, Buffer++) {
    Pair[1] *= 16;
    char C = *Buffer;
    if (C >= '0' && C <= '9')
      Pair[1] += C-'0';
    else if (C >= 'A' && C <= 'F')
      Pair[1] += C-'A'+10;
    else if (C >= 'a' && C <= 'f')
      Pair[1] += C-'a'+10;
  }
  if (Buffer != End)
    Error("constant bigger than 128 bits detected!");
}

/// FP80HexToIntPair - translate an 80 bit FP80 number (20 hexits) into
/// { low64, high16 } as usual for an APInt.
void LLLexer::FP80HexToIntPair(const char *Buffer, const char *End,
                           uint64_t Pair[2]) {
  Pair[1] = 0;
  for (int i=0; i<4 && Buffer != End; i++, Buffer++) {
    assert(Buffer != End);
    Pair[1] *= 16;
    char C = *Buffer;
    if (C >= '0' && C <= '9')
      Pair[1] += C-'0';
    else if (C >= 'A' && C <= 'F')
      Pair[1] += C-'A'+10;
    else if (C >= 'a' && C <= 'f')
      Pair[1] += C-'a'+10;
  }
  Pair[0] = 0;
  for (int i=0; i<16; i++, Buffer++) {
    Pair[0] *= 16;
    char C = *Buffer;
    if (C >= '0' && C <= '9')
      Pair[0] += C-'0';
    else if (C >= 'A' && C <= 'F')
      Pair[0] += C-'A'+10;
    else if (C >= 'a' && C <= 'f')
      Pair[0] += C-'a'+10;
  }
  if (Buffer != End)
    Error("constant bigger than 128 bits detected!");
}

// UnEscapeLexed - Run through the specified buffer and change \xx codes to the
// appropriate character.
static void UnEscapeLexed(std::string &Str) {
  if (Str.empty()) return;

  char *Buffer = &Str[0], *EndBuffer = Buffer+Str.size();
  char *BOut = Buffer;
  for (char *BIn = Buffer; BIn != EndBuffer; ) {
    if (BIn[0] == '\\') {
      if (BIn < EndBuffer-1 && BIn[1] == '\\') {
        *BOut++ = '\\'; // Two \ becomes one
        BIn += 2;
      } else if (BIn < EndBuffer-2 && isxdigit(BIn[1]) && isxdigit(BIn[2])) {
        char Tmp = BIn[3]; BIn[3] = 0;      // Terminate string
        *BOut = (char)strtol(BIn+1, 0, 16); // Convert to number
        BIn[3] = Tmp;                       // Restore character
        BIn += 3;                           // Skip over handled chars
        ++BOut;
      } else {
        *BOut++ = *BIn++;
      }
    } else {
      *BOut++ = *BIn++;
    }
  }
  Str.resize(BOut-Buffer);
}

/// isLabelChar - Return true for [-a-zA-Z$._0-9].
static bool isLabelChar(char C) {
  return isalnum(C) || C == '-' || C == '$' || C == '.' || C == '_';
}


/// isLabelTail - Return true if this pointer points to a valid end of a label.
static const char *isLabelTail(const char *CurPtr) {
  while (1) {
    if (CurPtr[0] == ':') return CurPtr+1;
    if (!isLabelChar(CurPtr[0])) return 0;
    ++CurPtr;
  }
}



//===----------------------------------------------------------------------===//
// Lexer definition.<commit_msg>LLLexer: Factor hex char parsing.<commit_after>//===- LLLexer.cpp - Lexer for .ll Files ----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implement the Lexer for .ll files.
//
//===----------------------------------------------------------------------===//

#include "LLLexer.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Instruction.h"
#include "llvm/LLVMContext.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Assembly/Parser.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/raw_ostream.h"
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <cstring>
using namespace llvm;

bool LLLexer::Error(LocTy ErrorLoc, const Twine &Msg) const {
  ErrorInfo = SM.GetMessage(ErrorLoc, SourceMgr::DK_Error, Msg);
  return true;
}

//===----------------------------------------------------------------------===//
// Helper functions.
//===----------------------------------------------------------------------===//

// atoull - Convert an ascii string of decimal digits into the unsigned long
// long representation... this does not have to do input error checking,
// because we know that the input will be matched by a suitable regex...
//
uint64_t LLLexer::atoull(const char *Buffer, const char *End) {
  uint64_t Result = 0;
  for (; Buffer != End; Buffer++) {
    uint64_t OldRes = Result;
    Result *= 10;
    Result += *Buffer-'0';
    if (Result < OldRes) {  // Uh, oh, overflow detected!!!
      Error("constant bigger than 64 bits detected!");
      return 0;
    }
  }
  return Result;
}

static char parseHexChar(char C) {
  if (C >= '0' && C <= '9')
    return C-'0';
  if (C >= 'A' && C <= 'F')
    return C-'A'+10;
  if (C >= 'a' && C <= 'f')
    return C-'a'+10;
  return 0;
}

uint64_t LLLexer::HexIntToVal(const char *Buffer, const char *End) {
  uint64_t Result = 0;
  for (; Buffer != End; ++Buffer) {
    uint64_t OldRes = Result;
    Result *= 16;
    Result += parseHexChar(*Buffer);

    if (Result < OldRes) {   // Uh, oh, overflow detected!!!
      Error("constant bigger than 64 bits detected!");
      return 0;
    }
  }
  return Result;
}

void LLLexer::HexToIntPair(const char *Buffer, const char *End,
                           uint64_t Pair[2]) {
  Pair[0] = 0;
  for (int i=0; i<16; i++, Buffer++) {
    assert(Buffer != End);
    Pair[0] *= 16;
    Pair[0] += parseHexChar(*Buffer);
  }
  Pair[1] = 0;
  for (int i=0; i<16 && Buffer != End; i++, Buffer++) {
    Pair[1] *= 16;
    Pair[1] += parseHexChar(*Buffer);
  }
  if (Buffer != End)
    Error("constant bigger than 128 bits detected!");
}

/// FP80HexToIntPair - translate an 80 bit FP80 number (20 hexits) into
/// { low64, high16 } as usual for an APInt.
void LLLexer::FP80HexToIntPair(const char *Buffer, const char *End,
                           uint64_t Pair[2]) {
  Pair[1] = 0;
  for (int i=0; i<4 && Buffer != End; i++, Buffer++) {
    assert(Buffer != End);
    Pair[1] *= 16;
    Pair[1] += parseHexChar(*Buffer);
  }
  Pair[0] = 0;
  for (int i=0; i<16; i++, Buffer++) {
    Pair[0] *= 16;
    Pair[0] += parseHexChar(*Buffer);
  }
  if (Buffer != End)
    Error("constant bigger than 128 bits detected!");
}

// UnEscapeLexed - Run through the specified buffer and change \xx codes to the
// appropriate character.
static void UnEscapeLexed(std::string &Str) {
  if (Str.empty()) return;

  char *Buffer = &Str[0], *EndBuffer = Buffer+Str.size();
  char *BOut = Buffer;
  for (char *BIn = Buffer; BIn != EndBuffer; ) {
    if (BIn[0] == '\\') {
      if (BIn < EndBuffer-1 && BIn[1] == '\\') {
        *BOut++ = '\\'; // Two \ becomes one
        BIn += 2;
      } else if (BIn < EndBuffer-2 && isxdigit(BIn[1]) && isxdigit(BIn[2])) {
        *BOut = parseHexChar(BIn[1]) * 16 + parseHexChar(BIn[2]);
        BIn += 3;                           // Skip over handled chars
        ++BOut;
      } else {
        *BOut++ = *BIn++;
      }
    } else {
      *BOut++ = *BIn++;
    }
  }
  Str.resize(BOut-Buffer);
}

/// isLabelChar - Return true for [-a-zA-Z$._0-9].
static bool isLabelChar(char C) {
  return isalnum(C) || C == '-' || C == '$' || C == '.' || C == '_';
}


/// isLabelTail - Return true if this pointer points to a valid end of a label.
static const char *isLabelTail(const char *CurPtr) {
  while (1) {
    if (CurPtr[0] == ':') return CurPtr+1;
    if (!isLabelChar(CurPtr[0])) return 0;
    ++CurPtr;
  }
}



//===----------------------------------------------------------------------===//
// Lexer definition.
{'fraction_non_alphanumeric': 0.2509641342074817, 'fraction_numerical': 0.025742383339760894, 'mean_word_length': 1.9535876993166288, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5970102', 'n_tokens_mistral': 3728, 'n_tokens_neox': 3490, 'n_words': 1004}