<commit_before>// file      : cli/token.hxx
// author    : <NAME> <<EMAIL>>
// copyright : Copyright (c) 2009 Code Synthesis Tools CC
// license   : MIT; see accompanying LICENSE file

#ifndef CLI_TOKEN_HXX
#define CLI_TOKEN_HXX

#include <string>
#include <cstddef>

class Token
{
public:
  enum Type
  {
    t_eos,
    t_keyword,
    t_identifier,
    t_punctuation,
    t_bracket_lit,
    t_string_lit,
    t_bool_lit,
    t_int_lit,
    t_float_lit
  };

  Type
  type () const
  {
    return type_;
  }

  size_t
  line () const
  {
    return l_;
  }

  size_t
  column () const
  {
    return c_;
  }

  // Keyword
  //
public:
  enum Keyword
  {
    k_include,
    k_namespace,
    k_class,
    k_bool
  };

  Keyword
  keyword () const
  {
    return keyword_;
  }

  // Identifier
  //
public:
  std::string const&
  identifier () const
  {
    return str_;
  }

  // Punctuation
  //
public:
  enum Punctuation
  {
    p_semi,
    p_comma,
    p_colon,
    p_lcbrace,
    p_rcbrace,
    p_lparen,
    p_rparen,
    p_eq,
    p_or
  };

  Punctuation
  punctuation () const
  {
    return punctuation_;
  }

  // Literals.
  //
public:
  std::string const&
  literal () const
  {
    return str_;
  }

  // C-tors.
  //
public:
  // EOS.
  //
  Token (size_t l, size_t c)
      : l_ (l), c_ (c), type_ (t_eos)
  {
  }

  Token (Keyword k, size_t l, size_t c)
      : l_ (l), c_ (c), type_ (t_keyword), keyword_ (k)
  {
  }

  Token (Punctuation p, size_t l, size_t c)
      : l_ (l), c_ (c), type_ (t_punctuation), punctuation_ (p)
  {
  }

  // Identifier & literals.
  //
  Token (Type t, std::string const& s, size_t l, size_t c)
      : l_ (l), c_ (c), type_ (t), str_ (s)
  {
  }

private:
  size_t l_;
  size_t c_;

  Type type_;

  Keyword keyword_;
  Punctuation punctuation_;
  std::string str_;
};

#endif // CLI_TOKEN_HXX
<commit_msg>Add the rest of the keywords<commit_after>// file      : cli/token.hxx
// author    : <NAME> <<EMAIL>>
// copyright : Copyright (c) 2009 Code Synthesis Tools CC
// license   : MIT; see accompanying LICENSE file

#ifndef CLI_TOKEN_HXX
#define CLI_TOKEN_HXX

#include <string>
#include <cstddef> // std::size_t

class Token
{
public:
  enum Type
  {
    t_eos,
    t_keyword,
    t_identifier,
    t_punctuation,
    t_path_lit,
    t_string_lit,
    t_char_lit,
    t_bool_lit,
    t_int_lit,
    t_float_lit
  };

  Type
  type () const
  {
    return type_;
  }

  std::size_t
  line () const
  {
    return l_;
  }

  std::size_t
  column () const
  {
    return c_;
  }

  // Keyword
  //
public:
  enum Keyword
  {
    k_include,
    k_namespace,
    k_class,
    k_signed,
    k_unsigned,
    k_bool,
    k_char,
    k_wchar,
    k_short,
    k_int,
    k_long,
    k_float,
    k_double
  };

  Keyword
  keyword () const
  {
    return keyword_;
  }

  // Identifier
  //
public:
  std::string const&
  identifier () const
  {
    return str_;
  }

  // Punctuation
  //
public:
  enum Punctuation
  {
    p_semi,
    p_comma,
    p_colon,
    p_lcbrace,
    p_rcbrace,
    p_lparen,
    p_rparen,
    p_eq,
    p_or
  };

  Punctuation
  punctuation () const
  {
    return punctuation_;
  }

  // Literals.
  //
public:
  std::string const&
  literal () const
  {
    return str_;
  }

  // C-tors.
  //
public:
  // EOS.
  //
  Token (std::size_t l, std::size_t c)
      : l_ (l), c_ (c), type_ (t_eos)
  {
  }

  Token (Keyword k, std::size_t l, std::size_t c)
      : l_ (l), c_ (c), type_ (t_keyword), keyword_ (k)
  {
  }

  Token (Punctuation p, std::size_t l, std::size_t c)
      : l_ (l), c_ (c), type_ (t_punctuation), punctuation_ (p)
  {
  }

  // Identifier & literals.
  //
  Token (Type t, std::string const& s, std::size_t l, std::size_t c)
      : l_ (l), c_ (c), type_ (t), str_ (s)
  {
  }

private:
  std::size_t l_;
  std::size_t c_;

  Type type_;

  Keyword keyword_;
  Punctuation punctuation_;
  std::string str_;
};

#endif // CLI_TOKEN_HXX

{'fraction_non_alphanumeric': 0.12668532180106842, 'fraction_numerical': 0.002035105571101501, 'mean_word_length': 2.1206349206349207, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1818648', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1636, 'n_words': 382}