<commit_before>//
// Copyright (C) 2004 <NAME>
// All rights reserved.
// Licensed to the public under the terms of the GNU LGPL (>= 2),
// see the file COPYING for details.
//

#include "Synopsis/Lexer.hh"
#include "Synopsis/Buffer.hh"
#include <iostream>
#include <cassert>

using namespace Synopsis;

Lexer::Lexer(Buffer *buffer, int tokenset)
  : my_buffer(buffer),
    my_token(my_buffer->ptr(), 0, '\n')
{
  my_keywords["asm"] = Token::ATTRIBUTE;
  my_keywords["auto"] = Token::AUTO;
  my_keywords["break"] = Token::BREAK;
  my_keywords["case"] = Token::CASE;
  my_keywords["char"] = Token::CHAR;
  my_keywords["const"] = Token::CONST;
  my_keywords["continue"] = Token::CONTINUE;
  my_keywords["default"] = Token::DEFAULT;
  my_keywords["do"] = Token::DO;
  my_keywords["double"] = Token::DOUBLE;
  my_keywords["else"] = Token::ELSE;
  my_keywords["enum"] = Token::ENUM;
  my_keywords["extern"] = Token::EXTERN;
  my_keywords["float"] = Token::FLOAT;
  my_keywords["for"] = Token::FOR;
  my_keywords["goto"] = Token::GOTO;
  my_keywords["if"] = Token::IF;
  my_keywords["inline"] = Token::INLINE;
  my_keywords["int"] = Token::INT;
  my_keywords["long"] = Token::LONG;
  my_keywords["register"] = Token::REGISTER;
  my_keywords["return"] = Token::RETURN;
  my_keywords["short"] = Token::SHORT;
  my_keywords["signed"] = Token::SIGNED;
  my_keywords["sizeof"] = Token::SIZEOF;
  my_keywords["static"] = Token::STATIC;
  my_keywords["struct"] = Token::STRUCT;
  my_keywords["switch"] = Token::SWITCH;
  my_keywords["typedef"] = Token::TYPEDEF;
  my_keywords["union"] = Token::UNION;
  my_keywords["unsigned"] = Token::UNSIGNED;
  my_keywords["void"] = Token::VOID;
  my_keywords["volatile"] = Token::VOLATILE;
  my_keywords["wchar_t"] = Token::WCHAR;
  my_keywords["while"] = Token::WHILE;
  if (tokenset & CXX)
  {
    my_keywords["bool"] = Token::BOOLEAN;
    my_keywords["catch"] = Token::CATCH;
    my_keywords["class"] = Token::CLASS;
    my_keywords["delete"] = Token::DELETE;
    my_keywords["false"] = Token::Constant;
    my_keywords["friend"] = Token::FRIEND;
    my_keywords["mutable"] = Token::MUTABLE;
    my_keywords["namespace"] = Token::NAMESPACE;
    my_keywords["new"] = Token::NEW;
    my_keywords["operator"] = Token::OPERATOR;
    my_keywords["private"] = Token::PRIVATE;
    my_keywords["protected"] = Token::PROTECTED;
    my_keywords["public"] = Token::PUBLIC;
    my_keywords["template"] = Token::TEMPLATE;
    my_keywords["this"] = Token::THIS;
    my_keywords["throw"] = Token::THROW;
    my_keywords["true"] = Token::Constant;
    my_keywords["try"] = Token::TRY;
    my_keywords["typeid"] = Token::TYPEID;
    my_keywords["typename"] = Token::CLASS; // FIXME !!
    my_keywords["using"] = Token::USING;
    my_keywords["virtual"] = Token::VIRTUAL;
  }
  if (tokenset & GNU)
  {
    my_keywords["__alignof__"] = Token::SIZEOF;<commit_msg>only recognize 'wchar_t' when analyzing C++<commit_after>//
// Copyright (C) 2004 <NAME>
// All rights reserved.
// Licensed to the public under the terms of the GNU LGPL (>= 2),
// see the file COPYING for details.
//

#include "Synopsis/Lexer.hh"
#include "Synopsis/Buffer.hh"
#include <iostream>
#include <cassert>

using namespace Synopsis;

Lexer::Lexer(Buffer *buffer, int tokenset)
  : my_buffer(buffer),
    my_token(my_buffer->ptr(), 0, '\n')
{
  my_keywords["asm"] = Token::ATTRIBUTE;
  my_keywords["auto"] = Token::AUTO;
  my_keywords["break"] = Token::BREAK;
  my_keywords["case"] = Token::CASE;
  my_keywords["char"] = Token::CHAR;
  my_keywords["const"] = Token::CONST;
  my_keywords["continue"] = Token::CONTINUE;
  my_keywords["default"] = Token::DEFAULT;
  my_keywords["do"] = Token::DO;
  my_keywords["double"] = Token::DOUBLE;
  my_keywords["else"] = Token::ELSE;
  my_keywords["enum"] = Token::ENUM;
  my_keywords["extern"] = Token::EXTERN;
  my_keywords["float"] = Token::FLOAT;
  my_keywords["for"] = Token::FOR;
  my_keywords["goto"] = Token::GOTO;
  my_keywords["if"] = Token::IF;
  my_keywords["inline"] = Token::INLINE;
  my_keywords["int"] = Token::INT;
  my_keywords["long"] = Token::LONG;
  my_keywords["register"] = Token::REGISTER;
  my_keywords["return"] = Token::RETURN;
  my_keywords["short"] = Token::SHORT;
  my_keywords["signed"] = Token::SIGNED;
  my_keywords["sizeof"] = Token::SIZEOF;
  my_keywords["static"] = Token::STATIC;
  my_keywords["struct"] = Token::STRUCT;
  my_keywords["switch"] = Token::SWITCH;
  my_keywords["typedef"] = Token::TYPEDEF;
  my_keywords["union"] = Token::UNION;
  my_keywords["unsigned"] = Token::UNSIGNED;
  my_keywords["void"] = Token::VOID;
  my_keywords["volatile"] = Token::VOLATILE;
  my_keywords["while"] = Token::WHILE;
  if (tokenset & CXX)
  {
    my_keywords["bool"] = Token::BOOLEAN;
    my_keywords["catch"] = Token::CATCH;
    my_keywords["class"] = Token::CLASS;
    my_keywords["delete"] = Token::DELETE;
    my_keywords["false"] = Token::Constant;
    my_keywords["friend"] = Token::FRIEND;
    my_keywords["mutable"] = Token::MUTABLE;
    my_keywords["namespace"] = Token::NAMESPACE;
    my_keywords["new"] = Token::NEW;
    my_keywords["operator"] = Token::OPERATOR;
    my_keywords["private"] = Token::PRIVATE;
    my_keywords["protected"] = Token::PROTECTED;
    my_keywords["public"] = Token::PUBLIC;
    my_keywords["template"] = Token::TEMPLATE;
    my_keywords["this"] = Token::THIS;
    my_keywords["throw"] = Token::THROW;
    my_keywords["true"] = Token::Constant;
    my_keywords["try"] = Token::TRY;
    my_keywords["typeid"] = Token::TYPEID;
    my_keywords["typename"] = Token::CLASS; // FIXME !!
    my_keywords["using"] = Token::USING;
    my_keywords["virtual"] = Token::VIRTUAL;
    my_keywords["wchar_t"] = Token::WCHAR;
  }
  if (tokenset & GNU)
  {
    my_keywords["__alignof__"] = Token::SIZEOF;
{'fraction_non_alphanumeric': 0.1884308971250433, 'fraction_numerical': 0.0020782819535850364, 'mean_word_length': 5.459731543624161, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7019736', 'n_tokens_mistral': 2133, 'n_tokens_neox': 1986, 'n_words': 340}