<commit_before>// Copyright (c) 2015 <NAME>
// All rights reserved

#ifndef LINGO_LEXING_HPP
#define LINGO_LEXING_HPP

// The lexing module provides support for writing lexers
// by providing a number of generic scanners for different
// character sequences.
//
// These algorithms are based on a Lexer concept. Every
// lexer must expose a number of operations.

#include "lingo/string.hpp"
#include "lingo/location.hpp"
#include "lingo/token.hpp"
#include "lingo/algorithm.hpp"
#include "lingo/error.hpp"

namespace lingo
{

// -------------------------------------------------------------------------- //
//                             Character classes

// Returns true if c is white space.
inline bool
is_space(char c)
{
  return std::isspace(c);
}


// Returns true if c is alphabetical.
inline bool
is_alpha(char c)
{
  return std::isalpha(c);
}


// Returns true if c is a binary digit.
inline bool
is_binary_digit(char c)
{
  return c == '0' || c == '1';
}


// Returns true if c is an octal digit.
inline bool
is_octal_digit(char c)
{
  return '0' <= c && c <= '7';
}


// Returns true if c is a decimial digit.
inline bool
is_decimal_digit(char c)
{
  return std::isdigit(c);
}


// Returns true if c is a hexadecimal digit.
inline bool
is_hexadecimal_digit(char c)
{
  return std::isxdigit(c);
}


// Returns true if c starts an identifier. 
inline bool
is_ident_head(char c)
{
  return is_alpha(c) || c == '_';
}


// Returns true if c can appear in the rest of an identifier.
inline bool
is_ident_tail(char c)
{
  return is_ident_tail(c) || is_decimal_digit(c);
}


// -------------------------------------------------------------------------- //
//                             Integer lexers
//
// TODO: Conditionally support a character as a digit separator
// in integer values.
//
// Note that an integer value like '0x' is an error. Note that the
// lexers for prefixed integer values require the lexer to define 
// the following functions:
// 
//    cxt.on_expected(loc, t)
//    cxt.on_expected(loc, str)


// Lexically analyze a decimal integer. The current character
// is `s` must be in the set of decimal digits.
template<typename Lexer, typename Stream>
inline Result_type<Lexer>
lex_decimal_integer(Lexer& lex, Stream& s, Location loc)
{
  auto pred = [](Stream& s) { return next_element_if(s, is_decimal_digit); };<commit_msg>Fix infinite recursion in identifier lex.<commit_after>// Copyright (c) 2015 <NAME>
// All rights reserved

#ifndef LINGO_LEXING_HPP
#define LINGO_LEXING_HPP

// The lexing module provides support for writing lexers
// by providing a number of generic scanners for different
// character sequences.
//
// These algorithms are based on a Lexer concept. Every
// lexer must expose a number of operations.

#include "lingo/string.hpp"
#include "lingo/location.hpp"
#include "lingo/token.hpp"
#include "lingo/algorithm.hpp"
#include "lingo/error.hpp"

namespace lingo
{

// -------------------------------------------------------------------------- //
//                             Character classes

// Returns true if c is white space.
inline bool
is_space(char c)
{
  return std::isspace(c);
}


// Returns true if c is alphabetical.
inline bool
is_alpha(char c)
{
  return std::isalpha(c);
}


// Returns true if c is a binary digit.
inline bool
is_binary_digit(char c)
{
  return c == '0' || c == '1';
}


// Returns true if c is an octal digit.
inline bool
is_octal_digit(char c)
{
  return '0' <= c && c <= '7';
}


// Returns true if c is a decimial digit.
inline bool
is_decimal_digit(char c)
{
  return std::isdigit(c);
}


// Returns true if c is a hexadecimal digit.
inline bool
is_hexadecimal_digit(char c)
{
  return std::isxdigit(c);
}


// Returns true if c starts an identifier. 
inline bool
is_ident_head(char c)
{
  return is_alpha(c) || c == '_';
}


// Returns true if c can appear in the rest of an identifier.
inline bool
is_ident_tail(char c)
{
  return is_ident_head(c) || is_decimal_digit(c);
}


// -------------------------------------------------------------------------- //
//                             Integer lexers
//
// TODO: Conditionally support a character as a digit separator
// in integer values.
//
// Note that an integer value like '0x' is an error. Note that the
// lexers for prefixed integer values require the lexer to define 
// the following functions:
// 
//    cxt.on_expected(loc, t)
//    cxt.on_expected(loc, str)


// Lexically analyze a decimal integer. The current character
// is `s` must be in the set of decimal digits.
template<typename Lexer, typename Stream>
inline Result_type<Lexer>
lex_decimal_integer(Lexer& lex, Stream& s, Location loc)
{
  auto pred = [](Stream& s) { return next_element_if(s, is_decimal_digit); };
{'fraction_non_alphanumeric': 0.16694807935837908, 'fraction_numerical': 0.003799071338117349, 'mean_word_length': 2.481998530492285, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7413758', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1457, 'n_words': 529}