<commit_before>#ifndef GRUNE_SYMBOL_TRAITS_H
#define GRUNE_SYMBOL_TRAITS_H

#include <algorithm>
#include <boost/algorithm/string/join.hpp>
#include <list>
#include <string>

#include "grune/grune_fwd.hpp"

namespace grune
{

template<class T>
class symbol_traits;

template<class T>
bool is_terminal(const T& t)
{
    return symbol_traits<T>::is_terminal(t);
}

template<class T>
bool is_empty(const T& t)
{
    return symbol_traits<T>::is_empty(t);
}

template<class T>
std::string to_string(const T& t)
{
    return symbol_traits<T>::to_string(t);
}

template<class T>
std::string identifier(const T& t)
{
    return symbol_traits<T>::identifier(t);
}

template<class T>
std::string text(const T& t)
{
    return symbol_traits<T>::text(t);
}

template<class T>
struct symbol_traits;

template<class T>
struct default_symbol_traits
{
    static bool is_terminal(const T& t) { return t.is_terminal(); }
    static bool is_empty(const T& t) { return t.is_empty(); }
    static std::string to_string(const T& t) { return t.to_string(); }
    static std::string identifier(const T& t) { return t.identifier(); }
    static std::string text(const T& t) { return t.text(); }
};

template<class T>
struct basic_symbol_traits
{
    static bool is_terminal(const T& t) { return true; }
    static bool is_empty(const T& t) { return grune::text(t).empty(); }
    static std::string to_string(const T& t) { return "\"" + grune::text(t) + "\""; }
    static std::string identifier(const T& t) { return ""; }
    static std::string text(const T& t) { return ""; }
};

template<class T>
struct numeric_symbol_traits : 
    public basic_symbol_traits<T>
{
    static bool is_empty(T) { return false; }
    static std::string text(T v) { return std::to_string(v); }
};

template<>
struct symbol_traits<std::string> : 
    public basic_symbol_traits<std::string>
{
    static bool is_empty(const std::string& s) { return s.empty(); }
    static std::string text(const std::string& s) { return s; }
};

template<>
struct symbol_traits<int> : 
    public numeric_symbol_traits<int>
{
};

template<>
std::string to_string<sequence>(const sequence& s);
template<>
std::string text<sequence>(const sequence& s);<commit_msg>Remove unused basic_symbol_traits member.<commit_after>#ifndef GRUNE_SYMBOL_TRAITS_H
#define GRUNE_SYMBOL_TRAITS_H

#include <algorithm>
#include <boost/algorithm/string/join.hpp>
#include <list>
#include <string>

#include "grune/grune_fwd.hpp"

namespace grune
{

template<class T>
class symbol_traits;

template<class T>
bool is_terminal(const T& t)
{
    return symbol_traits<T>::is_terminal(t);
}

template<class T>
bool is_empty(const T& t)
{
    return symbol_traits<T>::is_empty(t);
}

template<class T>
std::string to_string(const T& t)
{
    return symbol_traits<T>::to_string(t);
}

template<class T>
std::string identifier(const T& t)
{
    return symbol_traits<T>::identifier(t);
}

template<class T>
std::string text(const T& t)
{
    return symbol_traits<T>::text(t);
}

template<class T>
struct symbol_traits;

template<class T>
struct default_symbol_traits
{
    static bool is_terminal(const T& t) { return t.is_terminal(); }
    static bool is_empty(const T& t) { return t.is_empty(); }
    static std::string to_string(const T& t) { return t.to_string(); }
    static std::string identifier(const T& t) { return t.identifier(); }
    static std::string text(const T& t) { return t.text(); }
};

template<class T>
struct basic_symbol_traits
{
    static bool is_terminal(const T& t) { return true; }
    static bool is_empty(const T& t) { return grune::text(t).empty(); }
    static std::string to_string(const T& t) { return "\"" + grune::text(t) + "\""; }
    static std::string identifier(const T& t) { return ""; }
};

template<class T>
struct numeric_symbol_traits : 
    public basic_symbol_traits<T>
{
    static bool is_empty(T) { return false; }
    static std::string text(T v) { return std::to_string(v); }
};

template<>
struct symbol_traits<std::string> : 
    public basic_symbol_traits<std::string>
{
    static bool is_empty(const std::string& s) { return s.empty(); }
    static std::string text(const std::string& s) { return s; }
};

template<>
struct symbol_traits<int> : 
    public numeric_symbol_traits<int>
{
};

template<>
std::string to_string<sequence>(const sequence& s);
template<>
std::string text<sequence>(const sequence& s);
{'fraction_non_alphanumeric': 0.14409682575930577, 'fraction_numerical': 0.0, 'mean_word_length': 4.983606557377049, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779025', 'n_tokens_mistral': 1552, 'n_tokens_neox': 1500, 'n_words': 378}