<commit_before>#include "lexer.h"

#include <string>

#include "lexerexception.h"
#include "log.h"
#include "scriptsource.h"
#include "token.h"

namespace peachy {

  Lexer::~Lexer() {
    logger->debug("Lexer destructor");
  }

  Token * Lexer::nextToken() {
    logger->debug("Lexer::nextToken()");

    Token * token;
    bool gotToken = false;

    while(!gotToken) {
      currentChar = currentLine[currentPos];
      switch(state) {
        case LEXER_COMPLETE:
	  logger->debug("In state LEXER_COMPLETE");
	  token = new Token(logger, TOKEN_EOF);
	  resetToken();
	  gotToken = true;
          break;
        case LEXER_DEFAULT:
          logger->debug("In state LEXER_DEFAULT");
          switch(currentChar) {
	    case ' ':
	    case '\t':
	      logger->debug("Whitespace");
	      consume(false);
	      break;
	    case '\n':
	    case '\r':
	    case NULL:
	      logger->debug("End of line");
	      resetLine();
	      break;
            default:
	      if(isNumeric(currentChar)) {
                logger->debug("Current char is a number");
		setState(LEXER_IN_NUMBER);
		consume(true);
	      } else if(isLetter(currentChar)) {
	        logger->debug("Current char is a letter");
                setState(LEXER_IN_IDENTIFIER);
                consume(true);
              } else if(isOperator(currentChar)) {
	        logger->debug("Current char is an operator");
		setState(LEXER_IN_OPERATOR);
		consume(true);
	      } else {
                throw LexerException(
		  std::string("Invalid character encountered: ").append(
		    1, currentChar));
	      }
	      break;
	  }
          break;
	case LEXER_IN_NUMBER:
	  logger->debug("In state LEXER_IN_NUMBER");
	  if(isNumeric(currentChar)) {
            logger->debug("Another digit of the current number");
	    consume(true);
	  } else {
            logger->debug("End of number");
            token = new Token(logger, TOKEN_NUMBER, currentSequence);
	    resetToken();
            gotToken = true;
	  }
	  break;
	case LEXER_IN_IDENTIFIER:
	  logger->debug("In state LEXER_IN_IDENTIFIER");
	  if(isIdentifier(currentChar)) {
            logger->debug("Another character of the current identifier");
	    consume(true);
	  } else {
            logger->debug("End of identifier");
	    if(isKeyword(currentSequence)) {
              token = new Token(logger, TOKEN_KEYWORD, currentSequence);
	    } else {
	      token = new Token(logger, TOKEN_IDENTIFIER, currentSequence);
	    }
	    resetToken();
	    gotToken = true;
	  }
	  break;
	case LEXER_IN_OPERATOR:
	  logger->debug("In state LEXER_IN_OPERATOR");
	  if(isOperator(currentChar)) {
            logger->debug("Another character of the current operator");
	    consume(true);
	  } else if(isNumeric(currentChar) &&
	          currentSequence.compare("-") == 0) {
            logger->debug("Looks like a negative number");
	    setState(LEXER_IN_NUMBER);
	    consume(true);
	  } else {
            logger->debug("End of operator");
	    token = new Token(logger, TOKEN_OPERATOR, currentSequence);
	    resetToken();
	    gotToken = true;
	  }
	  break;
        case LEXER_NEED_INPUT:
          logger->debug("In state LEXER_NEED_INPUT");
	  if(!scriptSource->hasMoreLines()) {
            logger->debug("End of input");
	    setState(LEXER_COMPLETE);
	  } else {
            setCurrentLine(scriptSource->getLine());
	    logger->debug("Got new line from script source");
	    resetToken();
	  }
          break;
	case LEXER_ERROR:
          logger->debug("In state LEXER_ERROR");
	  throw LexerException("Invalid lexer state");
          break;
	default:
          logger->debug("In an unknown state");<commit_msg>sort out indentation<commit_after>#include "lexer.h"

#include <string>

#include "lexerexception.h"
#include "log.h"
#include "scriptsource.h"
#include "token.h"

namespace peachy {

  Lexer::~Lexer() {
    logger->debug("Lexer destructor");
  }

  Token * Lexer::nextToken() {
    logger->debug("Lexer::nextToken()");

    Token * token;
    bool gotToken = false;

    while(!gotToken) {
      currentChar = currentLine[currentPos];
      switch(state) {
        case LEXER_COMPLETE:
          logger->debug("In state LEXER_COMPLETE");
          token = new Token(logger, TOKEN_EOF);
          resetToken();
          gotToken = true;
          break;
        case LEXER_DEFAULT:
          logger->debug("In state LEXER_DEFAULT");
          switch(currentChar) {
            case ' ':
            case '\t':
              logger->debug("Whitespace");
              consume(false);
              break;
            case '\n':
            case '\r':
            case NULL:
              logger->debug("End of line");
              resetLine();
              break;
            default:
              if(isNumeric(currentChar)) {
                logger->debug("Current char is a number");
                setState(LEXER_IN_NUMBER);
                consume(true);
              } else if(isLetter(currentChar)) {
                logger->debug("Current char is a letter");
                setState(LEXER_IN_IDENTIFIER);
                consume(true);
              } else if(isOperator(currentChar)) {
                logger->debug("Current char is an operator");
                setState(LEXER_IN_OPERATOR);
                consume(true);
              } else {
                throw LexerException(
                  std::string("Invalid character encountered: ").append(
                    1, currentChar));
              }
              break;
          }
          break;
        case LEXER_IN_NUMBER:
          logger->debug("In state LEXER_IN_NUMBER");
          if(isNumeric(currentChar)) {
            logger->debug("Another digit of the current number");
            consume(true);
          } else {
            logger->debug("End of number");
            token = new Token(logger, TOKEN_NUMBER, currentSequence);
            resetToken();
            gotToken = true;
          }
          break;
        case LEXER_IN_IDENTIFIER:
          logger->debug("In state LEXER_IN_IDENTIFIER");
          if(isIdentifier(currentChar)) {
            logger->debug("Another character of the current identifier");
            consume(true);
          } else {
            logger->debug("End of identifier");
            if(isKeyword(currentSequence)) {
              token = new Token(logger, TOKEN_KEYWORD, currentSequence);
            } else {
              token = new Token(logger, TOKEN_IDENTIFIER, currentSequence);
            }
            resetToken();
            gotToken = true;
          }
          break;
        case LEXER_IN_OPERATOR:
          logger->debug("In state LEXER_IN_OPERATOR");
          if(isOperator(currentChar)) {
            logger->debug("Another character of the current operator");
            consume(true);
          } else if(isNumeric(currentChar) &&
            currentSequence.compare("-") == 0) {
            logger->debug("Looks like a negative number");
            setState(LEXER_IN_NUMBER);
            consume(true);
          } else {
            logger->debug("End of operator");
            token = new Token(logger, TOKEN_OPERATOR, currentSequence);
            resetToken();
            gotToken = true;
          }
          break;
        case LEXER_NEED_INPUT:
          logger->debug("In state LEXER_NEED_INPUT");
          if(!scriptSource->hasMoreLines()) {
            logger->debug("End of input");
            setState(LEXER_COMPLETE);
          } else {
            setCurrentLine(scriptSource->getLine());
            logger->debug("Got new line from script source");
            resetToken();
          }
          break;
        case LEXER_ERROR:
          logger->debug("In state LEXER_ERROR");
          throw LexerException("Invalid lexer state");
          break;
        default:
          logger->debug("In an unknown state");
{'fraction_non_alphanumeric': 0.11850152905198777, 'fraction_numerical': 0.0005096839959225281, 'mean_word_length': 1.9821428571428572, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793686', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2177, 'n_words': 469}