<commit_before>#ifndef PARSE_REFERENCE_HPP_
#define PARSE_REFERENCE_HPP_

#include "symbol.hpp"

#include <boost/optional.hpp>

namespace parse_reference_detail
{

inline bool is_letter(char c)
{
    return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
}
inline bool is_alpha_numeric(char c)
{
    return is_letter(c) || ('0' <= c && c <= '9');
}
inline bool is_operator(char c)
{<commit_msg>allow '_' in ref symbols (hacky)<commit_after>#ifndef PARSE_REFERENCE_HPP_
#define PARSE_REFERENCE_HPP_

#include "symbol.hpp"

#include <boost/optional.hpp>

namespace parse_reference_detail
{

inline bool is_letter(char c)
{
    return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
}
inline bool is_alpha_numeric(char c)
{
    return is_letter(c) || ('0' <= c && c <= '9');
}
inline bool is_operator(char c)
{
{'fraction_non_alphanumeric': 0.1822600243013366, 'fraction_numerical': 0.004860267314702308, 'mean_word_length': 4.6438356164383565, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626807', 'n_tokens_mistral': 322, 'n_tokens_neox': 316, 'n_words': 79}