<commit_before>#include "lexersuite.h"

#include <iostream>
#include <string>

#include "lexer.h"
#include "lexerexception.h"
#include "log.h"
#include "nullostream.h"
#include "scriptsource.h"
#include "stringscriptsource.h"
#include "testsuite.h"
#include "token.h"
#include "tokenfactory.h"

namespace peachy {

  namespace test {

    LexerSuite::~LexerSuite() {
      teardown();
    }

    void LexerSuite::startup() {
      ostream = new NullOStream();
      logger = new Log(ostream);
      tokenFactory = new TokenFactory(logger, logger);
    }

    void LexerSuite::teardown() {
      delete token;
      delete lexer;
      delete scriptSource;
      delete tokenFactory;
      delete logger;
    }

    void LexerSuite::run() {

      scriptSource = new StringScriptSource(logger, std::string(""));
      lexer = new Lexer(logger, tokenFactory, scriptSource);
      token = lexer->nextToken();
      ASSERT_EQUALS(token->getTokenType() == TOKEN_EOF, "First token is TOKEN_EOF");

      scriptSource = new StringScriptSource(logger, std::string("a"));
      lexer = new Lexer(logger, tokenFactory, scriptSource);
      token = lexer->nextToken();
      ASSERT_EQUALS(token->getTokenType() == TOKEN_IDENTIFIER, "First token is TOKEN_IDENTIFIER");
      ASSERT_EQUALS(token->getData().compare(std::string("a")) == 0, "First token is 'a'");
      
      scriptSource = new StringScriptSource(logger, std::string("<-"));<commit_msg>Delete ostream after lexer suite finishes<commit_after>#include "lexersuite.h"

#include <iostream>
#include <string>

#include "lexer.h"
#include "lexerexception.h"
#include "log.h"
#include "nullostream.h"
#include "scriptsource.h"
#include "stringscriptsource.h"
#include "testsuite.h"
#include "token.h"
#include "tokenfactory.h"

namespace peachy {

  namespace test {

    LexerSuite::~LexerSuite() {
      teardown();
    }

    void LexerSuite::startup() {
      ostream = new NullOStream();
      logger = new Log(ostream);
      tokenFactory = new TokenFactory(logger, logger);
    }

    void LexerSuite::teardown() {
      delete token;
      delete lexer;
      delete scriptSource;
      delete tokenFactory;
      delete logger;
      delete ostream;
    }

    void LexerSuite::run() {

      scriptSource = new StringScriptSource(logger, std::string(""));
      lexer = new Lexer(logger, tokenFactory, scriptSource);
      token = lexer->nextToken();
      ASSERT_EQUALS(token->getTokenType() == TOKEN_EOF, "First token is TOKEN_EOF");

      scriptSource = new StringScriptSource(logger, std::string("a"));
      lexer = new Lexer(logger, tokenFactory, scriptSource);
      token = lexer->nextToken();
      ASSERT_EQUALS(token->getTokenType() == TOKEN_IDENTIFIER, "First token is TOKEN_IDENTIFIER");
      ASSERT_EQUALS(token->getData().compare(std::string("a")) == 0, "First token is 'a'");
      
      scriptSource = new StringScriptSource(logger, std::string("<-"));
{'fraction_non_alphanumeric': 0.13831967213114754, 'fraction_numerical': 0.0006830601092896175, 'mean_word_length': 3.7859477124183005, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2105592', 'n_tokens_mistral': 942, 'n_tokens_neox': 909, 'n_words': 206}