<commit_before>//===-- EDToken.cpp - LLVM Enhanced Disassembler --------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
// 
//===----------------------------------------------------------------------===//
//
// This file implements the Enhanced Disassembler library's token class.  The
// token is responsible for vending information about the token, such as its
// type and logical value.
//
//===----------------------------------------------------------------------===//

#include "EDToken.h"
#include "EDDisassembler.h"
#include "llvm/MC/MCParser/MCAsmLexer.h"
#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
#include "llvm/ADT/SmallVector.h"
using namespace llvm;

EDToken::EDToken(StringRef str,
                 enum tokenType type,
                 uint64_t localType,
                 EDDisassembler &disassembler) :
  Disassembler(disassembler),
  Str(str),
  Type(type),
  LocalType(localType),
  OperandID(-1) {
}

EDToken::~EDToken() {
}

void EDToken::makeLiteral(bool sign, uint64_t absoluteValue) {
  Type = kTokenLiteral;
  LiteralSign = sign;
  LiteralAbsoluteValue = absoluteValue;
}

void EDToken::makeRegister(unsigned registerID) {
  Type = kTokenRegister;
  RegisterID = registerID;
}

void EDToken::setOperandID(int operandID) {<commit_msg>apparently also missing an include.<commit_after>//===-- EDToken.cpp - LLVM Enhanced Disassembler --------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
// 
//===----------------------------------------------------------------------===//
//
// This file implements the Enhanced Disassembler library's token class.  The
// token is responsible for vending information about the token, such as its
// type and logical value.
//
//===----------------------------------------------------------------------===//

#include "EDToken.h"
#include "EDDisassembler.h"
#include "llvm/MC/MCParser/MCAsmLexer.h"
#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/System/DataTypes.h"
using namespace llvm;

EDToken::EDToken(StringRef str,
                 enum tokenType type,
                 uint64_t localType,
                 EDDisassembler &disassembler) :
  Disassembler(disassembler),
  Str(str),
  Type(type),
  LocalType(localType),
  OperandID(-1) {
}

EDToken::~EDToken() {
}

void EDToken::makeLiteral(bool sign, uint64_t absoluteValue) {
  Type = kTokenLiteral;
  LiteralSign = sign;
  LiteralAbsoluteValue = absoluteValue;
}

void EDToken::makeRegister(unsigned registerID) {
  Type = kTokenRegister;
  RegisterID = registerID;
}

void EDToken::setOperandID(int operandID) {
{'fraction_non_alphanumeric': 0.22518159806295399, 'fraction_numerical': 0.0034590107229332413, 'mean_word_length': 2.167579408543264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5504791', 'n_tokens_mistral': 817, 'n_tokens_neox': 824, 'n_words': 206}