<commit_before>#include <papyrus/parser/PapyrusLexer.h>

#include <cctype>
#include <map>

#include <CapricaConfig.h>

namespace caprica { namespace papyrus { namespace parser {

void PapyrusLexer::setTok(TokenType tp, int consumeChars) {
  cur = Token(tp);
  cur.line = lineNum;
  for (int i = 0; i < consumeChars; i++)
    strm.get();
}

void PapyrusLexer::setTok(Token& tok) {
  cur = tok;
  cur.line = lineNum;
}

PapyrusLexer::Token PapyrusLexer::peekToken(int distance) {
  auto oldCur = cur;
  auto oldLine = lineNum;
  auto oldPos = strm.tellg();

  for (int i = 0; i <= distance; i++)
    consume();

  auto newTok = cur;
  cur = oldCur;
  lineNum = oldLine;
  strm.seekg(oldPos);
  return newTok;
}

static std::map<std::string, TokenType, CaselessStringComparer> keywordMap {
  { "as", TokenType::kAs },
  { "auto", TokenType::kAuto },
  { "autoreadonly", TokenType::kAutoReadOnly },
  { "bool", TokenType::kBool },
  { "else", TokenType::kElse },
  { "elseif", TokenType::kElseIf },
  { "endevent", TokenType::kEndEvent },
  { "endfunction", TokenType::kEndFunction },
  { "endif", TokenType::kEndIf },
  { "endproperty", TokenType::kEndProperty },
  { "endstate", TokenType::kEndState },
  { "endwhile", TokenType::kEndWhile },
  { "event", TokenType::kEvent },
  { "extends", TokenType::kExtends },
  { "false", TokenType::kFalse },
  { "float", TokenType::kFloat },
  { "function", TokenType::kFunction },
  { "global", TokenType::kGlobal },
  { "if", TokenType::kIf },
  { "import", TokenType::kImport },
  { "int", TokenType::kInt },
  { "length", TokenType::kLength },
  { "native", TokenType::kNative },
  { "new", TokenType::kNew },
  { "none", TokenType::kNone },
  { "parent", TokenType::kParent },
  { "property", TokenType::kProperty },
  { "return", TokenType::kReturn },
  { "scriptname", TokenType::kScriptName },
  { "self", TokenType::kSelf },
  { "state", TokenType::kState },
  { "string", TokenType::kString },
  { "true", TokenType::kTrue },
  { "while", TokenType::kWhile },
};

// Additional speculative keywords for FO4
static std::map<std::string, TokenType, CaselessStringComparer> speculativeKeywordMap {
  { "const", TokenType::kConst },
  { "endpropertygroup", TokenType::kEndPropertyGroup },
  { "endstruct", TokenType::kEndStruct },
  { "is", TokenType::kIs },
  { "propertygroup", TokenType::kPropertyGroup },
  { "struct", TokenType::kStruct },
  { "var", TokenType::kVar },
};

void PapyrusLexer::consume() {
StartOver:
  auto c = strm.get();
  
  switch (c) {
    case -1:
      return setTok(TokenType::END);
    case '(':
      return setTok(TokenType::LParen);
    case ')':
      return setTok(TokenType::RParen);
    case '[':
      return setTok(TokenType::LSquare);
    case ']':
      return setTok(TokenType::RSquare);
    case '.':
      return setTok(TokenType::Dot);
    case ',':
      return setTok(TokenType::Comma);

    case '=':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpEq, 1);
      return setTok(TokenType::Equal);
    case '!':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpNeq, 1);
      return setTok(TokenType::Exclaim);
    case '+':
      if (strm.peek() == '=')
        return setTok(TokenType::PlusEqual, 1);
      return setTok(TokenType::Plus);
    case '-':
      if (strm.peek() == '=')
        return setTok(TokenType::MinusEqual, 1);
      if (isdigit(strm.peek()))
        goto Number;
      return setTok(TokenType::Minus);
    case '*':
      if (strm.peek() == '=')
        return setTok(TokenType::MulEqual, 1);
      return setTok(TokenType::Mul);
    case '/':
      if (strm.peek() == '=')
        return setTok(TokenType::DivEqual, 1);
      return setTok(TokenType::Div);
    case '%':
      if (strm.peek() == '=')
        return setTok(TokenType::ModEqual, 1);
      return setTok(TokenType::Mod);
    case '<':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpLte, 1);
      return setTok(TokenType::CmpLt);
    case '>':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpGte, 1);
      return setTok(TokenType::CmpGt);

    case '|':
      if (strm.peek() != '|')
        fatalError("Bitwise OR is unsupported. Did you intend to use a logical or (\"||\") instead?");
      return setTok(TokenType::BooleanOr, 1);
    case '&':
      if (strm.peek() != '&')
        fatalError("Bitwise AND is unsupported. Did you intend to use a logical and (\"&&\") instead?");
      return setTok(TokenType::BooleanAnd, 1);

    Number:
    case '0':
    case '1':
    case '2':
    case '3':
    case '4':
    case '5':
    case '6':
    case '7':
    case '8':
    case '9':
    {
      std::ostringstream str;
      str.put(c);

      // It's hex.
      if (c == '0' && strm.peek() == 'x') {
        str.put(strm.get());
        while (isxdigit(strm.peek()))
          str.put(strm.get());
        
        auto i = std::stoul(str.str(), nullptr, 16);
        auto tok = Token(TokenType::Integer);
        tok.iValue = (int32_t)i;
        return setTok(tok);
      }

      // Either normal int or float.
      while (isdigit(strm.peek()))
        str.put(strm.get());

      // It's a float.
      if (strm.peek() == '.') {
        str.put(strm.get());
        while (isdigit(strm.peek()))
          str.put(strm.get());

        auto f = std::stof(str.str());
        auto tok = Token(TokenType::Float);
        tok.fValue = f;
        return setTok(tok);
      }

      // It's an integer.
      auto i = std::stoul(str.str());
      auto tok = Token(TokenType::Integer);
      tok.iValue = (int32_t)i;
      return setTok(tok);
    }

    case '_':
    case 'a':
    case 'b':
    case 'c':
    case 'd':
    case 'e':
    case 'f':
    case 'g':
    case 'h':
    case 'i':
    case 'j':
    case 'k':
    case 'l':
    case 'm':
    case 'n':
    case 'o':
    case 'p':
    case 'q':
    case 'r':
    case 's':
    case 't':
    case 'u':
    case 'v':
    case 'w':<commit_msg>Support lexing very large integers as floats.<commit_after>#include <papyrus/parser/PapyrusLexer.h>

#include <cctype>
#include <map>

#include <CapricaConfig.h>

namespace caprica { namespace papyrus { namespace parser {

void PapyrusLexer::setTok(TokenType tp, int consumeChars) {
  cur = Token(tp);
  cur.line = lineNum;
  for (int i = 0; i < consumeChars; i++)
    strm.get();
}

void PapyrusLexer::setTok(Token& tok) {
  cur = tok;
  cur.line = lineNum;
}

PapyrusLexer::Token PapyrusLexer::peekToken(int distance) {
  auto oldCur = cur;
  auto oldLine = lineNum;
  auto oldPos = strm.tellg();

  for (int i = 0; i <= distance; i++)
    consume();

  auto newTok = cur;
  cur = oldCur;
  lineNum = oldLine;
  strm.seekg(oldPos);
  return newTok;
}

static std::map<std::string, TokenType, CaselessStringComparer> keywordMap {
  { "as", TokenType::kAs },
  { "auto", TokenType::kAuto },
  { "autoreadonly", TokenType::kAutoReadOnly },
  { "bool", TokenType::kBool },
  { "else", TokenType::kElse },
  { "elseif", TokenType::kElseIf },
  { "endevent", TokenType::kEndEvent },
  { "endfunction", TokenType::kEndFunction },
  { "endif", TokenType::kEndIf },
  { "endproperty", TokenType::kEndProperty },
  { "endstate", TokenType::kEndState },
  { "endwhile", TokenType::kEndWhile },
  { "event", TokenType::kEvent },
  { "extends", TokenType::kExtends },
  { "false", TokenType::kFalse },
  { "float", TokenType::kFloat },
  { "function", TokenType::kFunction },
  { "global", TokenType::kGlobal },
  { "if", TokenType::kIf },
  { "import", TokenType::kImport },
  { "int", TokenType::kInt },
  { "length", TokenType::kLength },
  { "native", TokenType::kNative },
  { "new", TokenType::kNew },
  { "none", TokenType::kNone },
  { "parent", TokenType::kParent },
  { "property", TokenType::kProperty },
  { "return", TokenType::kReturn },
  { "scriptname", TokenType::kScriptName },
  { "self", TokenType::kSelf },
  { "state", TokenType::kState },
  { "string", TokenType::kString },
  { "true", TokenType::kTrue },
  { "while", TokenType::kWhile },
};

// Additional speculative keywords for FO4
static std::map<std::string, TokenType, CaselessStringComparer> speculativeKeywordMap {
  { "const", TokenType::kConst },
  { "endpropertygroup", TokenType::kEndPropertyGroup },
  { "endstruct", TokenType::kEndStruct },
  { "is", TokenType::kIs },
  { "propertygroup", TokenType::kPropertyGroup },
  { "struct", TokenType::kStruct },
  { "var", TokenType::kVar },
};

void PapyrusLexer::consume() {
StartOver:
  auto c = strm.get();
  
  switch (c) {
    case -1:
      return setTok(TokenType::END);
    case '(':
      return setTok(TokenType::LParen);
    case ')':
      return setTok(TokenType::RParen);
    case '[':
      return setTok(TokenType::LSquare);
    case ']':
      return setTok(TokenType::RSquare);
    case '.':
      return setTok(TokenType::Dot);
    case ',':
      return setTok(TokenType::Comma);

    case '=':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpEq, 1);
      return setTok(TokenType::Equal);
    case '!':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpNeq, 1);
      return setTok(TokenType::Exclaim);
    case '+':
      if (strm.peek() == '=')
        return setTok(TokenType::PlusEqual, 1);
      return setTok(TokenType::Plus);
    case '-':
      if (strm.peek() == '=')
        return setTok(TokenType::MinusEqual, 1);
      if (isdigit(strm.peek()))
        goto Number;
      return setTok(TokenType::Minus);
    case '*':
      if (strm.peek() == '=')
        return setTok(TokenType::MulEqual, 1);
      return setTok(TokenType::Mul);
    case '/':
      if (strm.peek() == '=')
        return setTok(TokenType::DivEqual, 1);
      return setTok(TokenType::Div);
    case '%':
      if (strm.peek() == '=')
        return setTok(TokenType::ModEqual, 1);
      return setTok(TokenType::Mod);
    case '<':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpLte, 1);
      return setTok(TokenType::CmpLt);
    case '>':
      if (strm.peek() == '=')
        return setTok(TokenType::CmpGte, 1);
      return setTok(TokenType::CmpGt);

    case '|':
      if (strm.peek() != '|')
        fatalError("Bitwise OR is unsupported. Did you intend to use a logical or (\"||\") instead?");
      return setTok(TokenType::BooleanOr, 1);
    case '&':
      if (strm.peek() != '&')
        fatalError("Bitwise AND is unsupported. Did you intend to use a logical and (\"&&\") instead?");
      return setTok(TokenType::BooleanAnd, 1);

    Number:
    case '0':
    case '1':
    case '2':
    case '3':
    case '4':
    case '5':
    case '6':
    case '7':
    case '8':
    case '9':
    {
      std::ostringstream str;
      str.put(c);

      // It's hex.
      if (c == '0' && strm.peek() == 'x') {
        str.put(strm.get());
        while (isxdigit(strm.peek()))
          str.put(strm.get());
        
        auto i = std::stoul(str.str(), nullptr, 16);
        auto tok = Token(TokenType::Integer);
        tok.iValue = (int32_t)i;
        return setTok(tok);
      }

      // Either normal int or float.
      while (isdigit(strm.peek()))
        str.put(strm.get());

      // It's a float.
      if (strm.peek() == '.') {
        str.put(strm.get());
        while (isdigit(strm.peek()))
          str.put(strm.get());

        auto f = std::stof(str.str());
        auto tok = Token(TokenType::Float);
        tok.fValue = f;
        return setTok(tok);
      }

      if (str.str().size() <= 8) {
        // It is probably an integer, but maybe not.
        try {
          auto i = std::stoul(str.str());
          auto tok = Token(TokenType::Integer);
          tok.iValue = (int32_t)i;
          return setTok(tok);
        } catch (std::out_of_range oor) { }
      }
      // It's very definitely a float, and a very large one at that.
      auto f = std::stof(str.str());
      auto tok = Token(TokenType::Float);
      tok.fValue = f;
      return setTok(tok);
    }

    case '_':
    case 'a':
    case 'b':
    case 'c':
    case 'd':
    case 'e':
    case 'f':
    case 'g':
    case 'h':
    case 'i':
    case 'j':
    case 'k':
    case 'l':
    case 'm':
    case 'n':
    case 'o':
    case 'p':
    case 'q':
    case 'r':
    case 's':
    case 't':
    case 'u':
    case 'v':
    case 'w':
{'fraction_non_alphanumeric': 0.18498287950432088, 'fraction_numerical': 0.005299201043534975, 'mean_word_length': 2.781442663378545, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1545054', 'n_tokens_mistral': 4458, 'n_tokens_neox': 4339, 'n_words': 925}