<commit_before>#ifndef TOML_PARSER
#define TOML_PARSER
#include <iostream>
#include <vector>
#include <string>
#include <utility>
#include <ctime>
#include "exceptions.hpp"
#include "escape_sequence.hpp"
#include "toml_values.hpp"
#include "is.hpp"
#include "utility.hpp"

namespace toml
{
template<typename charT>
inline bool is_whitespace(const charT c)
{// {{{
    return c == ' ' || c == '\t';
}// }}}

template<typename charT>
inline void skip_whitespace(std::basic_istream<charT>& is)
{// {{{
    while(is_whitespace(is.peek()) && !is.eof()){is.ignore();}
    return;
}// }}}

// if true, is.peek() should be '\n'.
template<typename charT>
bool is_newline(std::basic_istream<charT>& is)
{// {{{
    if(is.peek() == '\n')
    {
        return true;
    }
    else if(is.peek() == '\r')
    {
        const charT CR = is.get();
        if(is.peek() == '\n') return true;
        else is.putback(CR);
    }
    return false;
}// }}}

template<typename charT>
inline bool comment_starts(const charT c)
{//{{{
    return c == '#';
}//}}}

template<typename charT>
void skip_comment(std::basic_istream<charT>& is)
{//{{{
    if(not comment_starts(is.peek()))
        throw internal_error("invalid skip_comment call");

    // ignore until newline begin
    while(not is_newline(is) && not is.eof()){is.ignore();}

    if(!is.eof())// here, next charactor is '\n'
        is.ignore();
    return;
}//}}}

template<typename charT>
inline bool is_bare_key_component(const charT c)
{//{{{
         if('a' <= c && c <= 'z') return true;
    else if('A' <= c && c <= 'Z') return true;
    else if('0' <= c && c <= '9') return true;
    else if(c == '_' || c == '-') return true;
    return false;
}//}}}

template<typename charT>
std::basic_string<charT> read_bare_key(std::basic_istream<charT>& is)
{//{{{
    std::basic_string<charT> bare_key;
    while(is_bare_key_component(is.peek())){bare_key += is.get();}
    return bare_key;
}//}}}

template<typename charT>
std::basic_string<charT> read_array(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '[') throw internal_error("read_array: invalid call");

    std::basic_string<charT> retval;
    std::size_t counter = 0;
    while(true)
    {
        if(is.peek() == '[')
            ++counter;
        else if(is.peek() == ']')
            --counter;
        retval += is.get();

        if(counter == 0) return retval;
        if(is.eof()) throw syntax_error("non-closed array");
    }
}//}}}

template<typename charT>
std::basic_string<charT> read_inline_table(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '{')
        throw internal_error("read_inline_table: invalid call");

    std::basic_string<charT> retval;
    std::size_t counter = 0;
    while(true)
    {
        if(is.peek() == '{')
            ++counter;
        else if(is.peek() == '}')
            --counter;
        retval += is.get();

        if(counter == 0) return retval;
        if(is.eof()) throw syntax_error("non-closed inline table");
    }
}//}}}

template<typename charT>
std::basic_string<charT> read_basic_string(std::basic_istream<charT>& is)
{// {{{
    if(is.peek() != '\"')
        throw internal_error("read_basic_string: invalid call");

    std::basic_string<charT> retval;
    bool multi_line = false;
    retval += is.get();
    if(is.peek() == '\"')
    {
        retval += is.get();
        if(is.peek() != '\"') return retval; //here, retval == ""
        multi_line = true;
        retval += is.get(); //here, retval == """
    }

    std::size_t quote_count = 0; // for the end of multi line string
    bool escaping = false;
    while(true)
    {
        if(is.eof())
        {
            std::cerr << "is_multi_line " << multi_line << std::endl;
            throw syntax_error("read_basic_string: unexpected EOF");
        }
        switch(is.peek())
        {
            case '\\':
            {
                escaping = true;
                retval += is.get();
                quote_count = 0;
                break;
            }
            case '\"':
            {
                if(escaping)
                {
                    retval += is.get();
                    escaping = false;
                    break;
                }
                else if(multi_line)
                {
                    ++quote_count;
                    retval += is.get();
                    std::cerr << "now quote count = " << quote_count << std::endl;
                    if(quote_count == 3) return retval;
                    break;
                }
                else
                {
                    retval += is.get();
                    return retval;
                }
            }
            case '\n':
            {
                if(not multi_line)
                   throw syntax_error("read_basic_string: unexpected LF");
                else
                    retval += is.get();
                escaping = false;
                quote_count = 0;
                break;
            }
            default:
            {
                escaping = false;
                quote_count = 0;
                retval += is.get();
                break;
            }
        }
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_literal_string(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '\'')
        throw internal_error("read_literal_string: invalid call");

    std::basic_string<charT> retval;
    bool multi_line = false;
    retval += is.get();
    if(is.peek() == '\'')
    {
        retval += is.get();
        if(is.peek() != '\'') return retval; //here, retval == ''
        multi_line = true;
        retval += is.get(); //here, retval == '''
    }

    std::size_t quote_count = 0; // for the end of multi line string
    while(true)
    {
        if(is.eof())
            throw syntax_error("read_literal_string: unexpected EOF");
        switch(is.peek())
        {
            case '\'':
            {
                if(multi_line)
                {
                    ++quote_count;
                    retval += is.get();
                    if(quote_count == 3) return retval;
                    break;
                }
                else
                {
                    retval += is.get();
                    return retval;
                }
            }
            case '\n':
            {
                if(not multi_line)
                    throw syntax_error("read_literal_string: unexpected LF");
                else
                    retval += is.get();
                quote_count = 0;
                break;
            }
            default:
            {
                quote_count = 0;
                retval += is.get();
                break;
            }
        }
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_string(std::basic_istream<charT>& is)
{// {{{
    switch(is.peek())
    {
        case '\'':
            return read_literal_string(is);
        case '\"':
            return read_basic_string(is);
        default:
            throw internal_error("read_inline_table: invalid call");
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_simple_value(std::basic_istream<charT>& is)
{// {{{
    std::basic_string<charT> retval;
    std::cerr << "read_simple_value: ";
    while(not is_whitespace(is.peek()) && not is_newline(is) &&
          not comment_starts(is.peek()) && not is.eof() &&
          is.peek() != ',' && is.peek() != ']' && is.peek() != '}')
    {
        const charT c = is.get();
        std::cerr << c;
        retval += c;
    }
    std::cerr << std::endl;
    return retval;
}// }}}

template<typename charT>
std::basic_string<charT> read_value(std::basic_istream<charT>& is)
{//{{{
    switch(is.peek())
    {
        case '[':
            return read_array(is);
        case '{':
            return read_inline_table(is);
        case '\'':
            return read_string(is);
        case '\"':
            return read_string(is);
        default:
            return read_simple_value(is);
    }
}//}}}

template<typename charT>
std::basic_string<charT>
parse_literal_string_key(const std::basic_string<charT>& str)
{// {{{
    if(str.size() > 3 && str.substr(0, 3) == "\'\'\'")
    {
        if(str.at(4) == '\n')
        {
            return std::basic_string<charT>(str.begin()+4, str.end()-3);
        }
        else if(str.at(4) == '\r' && str.at(5) == '\n')
        {
            return std::basic_string<charT>(str.begin()+5, str.end()-3);
        }
        else
        {
            return std::basic_string<charT>(str.begin()+3, str.end()-3);
        }
    }
    else
    {
        return std::basic_string<charT>(str.begin()+1, str.end() - 1);
    }
}// }}}

template<typename charT>
std::basic_string<charT>
parse_multi_line_basic_string_key(const std::basic_string<charT>& str)
{// {{{
    std::basic_istringstream<charT> iss(
            std::basic_string<charT>(str.begin()+3, str.end()-3));
    if(is_newline(iss)) iss.ignore();

    std::basic_string<charT> contents;
    while(!iss.eof())
    {
        if(iss.peek() == '\\')
        {<commit_msg>fix parse_multi_line_literal_string<commit_after>#ifndef TOML_PARSER
#define TOML_PARSER
#include <iostream>
#include <vector>
#include <string>
#include <utility>
#include <ctime>
#include "exceptions.hpp"
#include "escape_sequence.hpp"
#include "toml_values.hpp"
#include "is.hpp"
#include "utility.hpp"

namespace toml
{
template<typename charT>
inline bool is_whitespace(const charT c)
{// {{{
    return c == ' ' || c == '\t';
}// }}}

template<typename charT>
inline void skip_whitespace(std::basic_istream<charT>& is)
{// {{{
    while(is_whitespace(is.peek()) && !is.eof()){is.ignore();}
    return;
}// }}}

// if true, is.peek() should be '\n'.
template<typename charT>
bool is_newline(std::basic_istream<charT>& is)
{// {{{
    if(is.peek() == '\n')
    {
        return true;
    }
    else if(is.peek() == '\r')
    {
        const charT CR = is.get();
        if(is.peek() == '\n') return true;
        else is.putback(CR);
    }
    return false;
}// }}}

template<typename charT>
inline bool comment_starts(const charT c)
{//{{{
    return c == '#';
}//}}}

template<typename charT>
void skip_comment(std::basic_istream<charT>& is)
{//{{{
    if(not comment_starts(is.peek()))
        throw internal_error("invalid skip_comment call");

    // ignore until newline begin
    while(not is_newline(is) && not is.eof()){is.ignore();}

    if(!is.eof())// here, next charactor is '\n'
        is.ignore();
    return;
}//}}}

template<typename charT>
inline bool is_bare_key_component(const charT c)
{//{{{
         if('a' <= c && c <= 'z') return true;
    else if('A' <= c && c <= 'Z') return true;
    else if('0' <= c && c <= '9') return true;
    else if(c == '_' || c == '-') return true;
    return false;
}//}}}

template<typename charT>
std::basic_string<charT> read_bare_key(std::basic_istream<charT>& is)
{//{{{
    std::basic_string<charT> bare_key;
    while(is_bare_key_component(is.peek())){bare_key += is.get();}
    return bare_key;
}//}}}

template<typename charT>
std::basic_string<charT> read_array(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '[') throw internal_error("read_array: invalid call");

    std::basic_string<charT> retval;
    std::size_t counter = 0;
    while(true)
    {
        if(is.peek() == '[')
            ++counter;
        else if(is.peek() == ']')
            --counter;
        retval += is.get();

        if(counter == 0) return retval;
        if(is.eof()) throw syntax_error("non-closed array");
    }
}//}}}

template<typename charT>
std::basic_string<charT> read_inline_table(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '{')
        throw internal_error("read_inline_table: invalid call");

    std::basic_string<charT> retval;
    std::size_t counter = 0;
    while(true)
    {
        if(is.peek() == '{')
            ++counter;
        else if(is.peek() == '}')
            --counter;
        retval += is.get();

        if(counter == 0) return retval;
        if(is.eof()) throw syntax_error("non-closed inline table");
    }
}//}}}

template<typename charT>
std::basic_string<charT> read_basic_string(std::basic_istream<charT>& is)
{// {{{
    if(is.peek() != '\"')
        throw internal_error("read_basic_string: invalid call");

    std::basic_string<charT> retval;
    bool multi_line = false;
    retval += is.get();
    if(is.peek() == '\"')
    {
        retval += is.get();
        if(is.peek() != '\"') return retval; //here, retval == ""
        multi_line = true;
        retval += is.get(); //here, retval == """
    }

    std::size_t quote_count = 0; // for the end of multi line string
    bool escaping = false;
    while(true)
    {
        if(is.eof())
        {
            std::cerr << "is_multi_line " << multi_line << std::endl;
            throw syntax_error("read_basic_string: unexpected EOF");
        }
        switch(is.peek())
        {
            case '\\':
            {
                escaping = true;
                retval += is.get();
                quote_count = 0;
                break;
            }
            case '\"':
            {
                if(escaping)
                {
                    retval += is.get();
                    escaping = false;
                    break;
                }
                else if(multi_line)
                {
                    ++quote_count;
                    retval += is.get();
                    std::cerr << "now quote count = " << quote_count << std::endl;
                    if(quote_count == 3) return retval;
                    break;
                }
                else
                {
                    retval += is.get();
                    return retval;
                }
            }
            case '\n':
            {
                if(not multi_line)
                   throw syntax_error("read_basic_string: unexpected LF");
                else
                    retval += is.get();
                escaping = false;
                quote_count = 0;
                break;
            }
            default:
            {
                escaping = false;
                quote_count = 0;
                retval += is.get();
                break;
            }
        }
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_literal_string(std::basic_istream<charT>& is)
{//{{{
    if(is.peek() != '\'')
        throw internal_error("read_literal_string: invalid call");

    std::basic_string<charT> retval;
    bool multi_line = false;
    retval += is.get();
    if(is.peek() == '\'')
    {
        retval += is.get();
        if(is.peek() != '\'') return retval; //here, retval == ''
        multi_line = true;
        retval += is.get(); //here, retval == '''
    }

    std::size_t quote_count = 0; // for the end of multi line string
    while(true)
    {
        if(is.eof())
            throw syntax_error("read_literal_string: unexpected EOF");
        switch(is.peek())
        {
            case '\'':
            {
                if(multi_line)
                {
                    ++quote_count;
                    retval += is.get();
                    if(quote_count == 3) return retval;
                    break;
                }
                else
                {
                    retval += is.get();
                    return retval;
                }
            }
            case '\n':
            {
                if(not multi_line)
                    throw syntax_error("read_literal_string: unexpected LF");
                else
                    retval += is.get();
                quote_count = 0;
                break;
            }
            default:
            {
                quote_count = 0;
                retval += is.get();
                break;
            }
        }
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_string(std::basic_istream<charT>& is)
{// {{{
    switch(is.peek())
    {
        case '\'':
            return read_literal_string(is);
        case '\"':
            return read_basic_string(is);
        default:
            throw internal_error("read_inline_table: invalid call");
    }
}// }}}

template<typename charT>
std::basic_string<charT> read_simple_value(std::basic_istream<charT>& is)
{// {{{
    std::basic_string<charT> retval;
    std::cerr << "read_simple_value: ";
    while(not is_whitespace(is.peek()) && not is_newline(is) &&
          not comment_starts(is.peek()) && not is.eof() &&
          is.peek() != ',' && is.peek() != ']' && is.peek() != '}')
    {
        const charT c = is.get();
        std::cerr << c;
        retval += c;
    }
    std::cerr << std::endl;
    return retval;
}// }}}

template<typename charT>
std::basic_string<charT> read_value(std::basic_istream<charT>& is)
{//{{{
    switch(is.peek())
    {
        case '[':
            return read_array(is);
        case '{':
            return read_inline_table(is);
        case '\'':
            return read_string(is);
        case '\"':
            return read_string(is);
        default:
            return read_simple_value(is);
    }
}//}}}

template<typename charT>
std::basic_string<charT>
parse_literal_string_key(const std::basic_string<charT>& str)
{// {{{
    if(str.size() > 6 && str.substr(0, 3) == "\'\'\'")
    {
        if(str.at(3) == '\n')
        {
            return std::basic_string<charT>(str.begin()+4, str.end()-3);
        }
        else if(str.at(3) == '\r' && str.at(4) == '\n')
        {
            return std::basic_string<charT>(str.begin()+5, str.end()-3);
        }
        else
        {
            return std::basic_string<charT>(str.begin()+3, str.end()-3);
        }
    }
    else
    {
        return std::basic_string<charT>(str.begin()+1, str.end() - 1);
    }
}// }}}

template<typename charT>
std::basic_string<charT>
parse_multi_line_basic_string_key(const std::basic_string<charT>& str)
{// {{{
    std::basic_istringstream<charT> iss(
            std::basic_string<charT>(str.begin()+3, str.end()-3));
    if(is_newline(iss)) iss.ignore();

    std::basic_string<charT> contents;
    while(!iss.eof())
    {
        if(iss.peek() == '\\')
        {
{'fraction_non_alphanumeric': 0.16810864089365896, 'fraction_numerical': 0.003395027926842624, 'mean_word_length': 1.759595043819885, 'pattern_counts': {'":': 0, '<': 165, '<?xml version=': 0, '>': 119, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7278175', 'n_tokens_mistral': 6026, 'n_tokens_neox': 5883, 'n_words': 1073}