<commit_before>#include "lexer.hpp"
#include "helper_lexer.hpp"

#include "gtest.h"

TEST(lexer, emptySource) {
  std::stringstream is;
  auto tokens = klang::tokenize(is);
  EXPECT_EQ("", test::to_string(tokens));
}

TEST(lexer, hello) {
  std::stringstream is;
  std::string nl("\n");
  is << "{~" << nl
     << "  とりあえずのデバッグ用" << nl
     << "  まだdeclareもstringもない" << nl
     << "  declare print(string) -> ();" << nl
     << "~}" << nl
     << "" << nl
     << "def main() -> (int) {" << nl
     << "  print(\"Hello, World!\\n\");" << nl
     << "  return 0;" << nl
     << "}" << nl;
  auto tokens = klang::tokenize(is);
  using T = klang::Token;
  using klang::TokenType;<commit_msg>Use raw string literals.<commit_after>#include "lexer.hpp"
#include "helper_lexer.hpp"

#include "gtest.h"

TEST(lexer, emptySource) {
  std::stringstream is;
  auto tokens = klang::tokenize(is);
  EXPECT_EQ("", test::to_string(tokens));
}

TEST(lexer, hello) {
  std::stringstream is;
  is <<
R"({~
  とりあえずのデバッグ用
  まだdeclareもstringもない
  declare print(string) -> ();
~}

def main() -> (int) {
  print("Hello, World!\n");
  return 0;
}
)";
  auto tokens = klang::tokenize(is);
  using T = klang::Token;
  using klang::TokenType;
{'fraction_non_alphanumeric': 0.20888157894736842, 'fraction_numerical': 0.001644736842105263, 'mean_word_length': 3.609848484848485, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3351454', 'n_tokens_mistral': 490, 'n_tokens_neox': 492, 'n_words': 93}