<commit_before>namespace cdmh {
namespace data_processing {
namespace detail {

template<typename T>
inline
unsigned const bit_count(T n)
{
    n = n - ((n >> 1) & 0x55555555);
    n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
    return (((n + (n >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
}

template<std::size_t N, typename... U>
using select_type = typename std::tuple_element<N, std::tuple<U...> >::type;

inline bool const isspace(char const ch)
{
    return ch == ' '  ||  ch == '\t'  ||  ch == '\r'  ||  ch == '\n';
}

template<typename It>
inline
void ltrim(It &it,It ite)
{
    while (it != ite  &&  isspace(*it))
        ++it;
}

template<typename It>
inline
void rtrim(It it,It &ite)
{
    std::reverse_iterator<It> rit(ite);
    std::reverse_iterator<It> rite(it);
    ltrim(rit, rite);
    ite = rit.base();
}

template<typename It>
inline
std::pair<std::pair<It,It>, type_mask_t>&
trim(std::pair<std::pair<It,It>, type_mask_t> &src)
{
    ltrim(src.first.first, src.first.second);
    rtrim(src.first.first, src.first.second);
    return src;
}

inline
std::pair<string_view, type_mask_t>
read_field(char const *&begin, char const *end)
{
    assert(begin != end);

    // we'll trim spaces before any quotes, but not within quotes
    ltrim(begin, end);

    bool in_quotes  = false;
    if (*begin == '\"')
    {
        in_quotes = true;
        ++begin;
    }

    auto it = begin;
    type_mask_t incl_type_mask = string_type;
    type_mask_t excl_type_mask = 0;

    // special case for unary operators
    if (*it == '-'  ||  *it == '+')
    {
        incl_type_mask |= double_type | integer_type;
        ++it;
    }

    bool expect_esc  = false;
    bool seen_period = false;
    bool seen_space  = false;
    for (; it!=end  &&  (in_quotes  ||  (!in_quotes  &&  *it != ','))  &&  !(in_quotes  &&  *it == '\"'  &&  !expect_esc); ++it)
    {
        if (*it >= '0'  &&  *it <= '9')<commit_msg>Optimized loop termination in read_field()<commit_after>namespace cdmh {
namespace data_processing {
namespace detail {

template<typename T>
inline
unsigned const bit_count(T n)
{
    n = n - ((n >> 1) & 0x55555555);
    n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
    return (((n + (n >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
}

template<std::size_t N, typename... U>
using select_type = typename std::tuple_element<N, std::tuple<U...> >::type;

inline bool const isspace(char const ch)
{
    return ch == ' '  ||  ch == '\t'  ||  ch == '\r'  ||  ch == '\n';
}

template<typename It>
inline
void ltrim(It &it,It ite)
{
    while (it != ite  &&  isspace(*it))
        ++it;
}

template<typename It>
inline
void rtrim(It it,It &ite)
{
    std::reverse_iterator<It> rit(ite);
    std::reverse_iterator<It> rite(it);
    ltrim(rit, rite);
    ite = rit.base();
}

template<typename It>
inline
std::pair<std::pair<It,It>, type_mask_t>&
trim(std::pair<std::pair<It,It>, type_mask_t> &src)
{
    ltrim(src.first.first, src.first.second);
    rtrim(src.first.first, src.first.second);
    return src;
}

inline
std::pair<string_view, type_mask_t>
read_field(char const *&begin, char const *end)
{
    assert(begin != end);

    // we'll trim spaces before any quotes, but not within quotes
    ltrim(begin, end);

    bool in_quotes  = false;
    if (*begin == '\"')
    {
        in_quotes = true;
        ++begin;
    }

    auto it = begin;
    type_mask_t incl_type_mask = string_type;
    type_mask_t excl_type_mask = 0;

    // special case for unary operators
    if (*it == '-'  ||  *it == '+')
    {
        incl_type_mask |= double_type | integer_type;
        ++it;
    }

    bool expect_esc  = false;
    bool seen_period = false;
    bool seen_space  = false;
    for (; it!=end  &&  (!in_quotes  ||  *it != '\"'  ||  expect_esc)  &&  (in_quotes  ||  *it != ','); ++it)
    {
        assert(
            ((in_quotes  ||  (!in_quotes  &&  *it != ','))  &&  !(in_quotes  &&  *it == '\"'  &&  !expect_esc))
            ==
            ((!in_quotes  ||  *it != '\"'  ||  expect_esc)  &&  (in_quotes  ||  *it != ','))
        );

        if (*it >= '0'  &&  *it <= '9')
{'fraction_non_alphanumeric': 0.18148510014655594, 'fraction_numerical': 0.023937469467513434, 'mean_word_length': 2.7951807228915664, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6917163', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1562, 'n_words': 356}