<commit_before>#pragma once

#include "unicorn/core.hpp"
#include "unicorn/character.hpp"
#include "unicorn/regex.hpp"
#include "unicorn/string.hpp"
#include <cstring>
#include <functional>
#include <memory>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <vector>

namespace Unicorn {

    // Forward declarations

    template <typename CX> class BasicLexer;
    template <typename CX> struct BasicToken;
    template <typename CX> class BasicTokenIterator;
    class SyntaxError;

    using Lexer = BasicLexer<char>;
    using Token = BasicToken<char>;
    using TokenIterator = BasicTokenIterator<char>;
    using ByteLexer = BasicLexer<ByteMode>;
    using ByteTokenIterator = BasicTokenIterator<ByteMode>;

    #if defined(UNICORN_PCRE16)
        using Lexer16 = BasicLexer<char16_t>;
        using Token16 = BasicToken<char16_t>;
        using TokenIterator16 = BasicTokenIterator<char16_t>;
    #endif

    #if defined(UNICORN_PCRE32)
        using Lexer32 = BasicLexer<char32_t>;
        using Token32 = BasicToken<char32_t>;
        using TokenIterator32 = BasicTokenIterator<char32_t>;
    #endif

    #if defined(UNICORN_PCRE_WCHAR)
        using WideLexer = BasicLexer<wchar_t>;
        using WideToken = BasicToken<wchar_t>;
        using WideTokenIterator = BasicTokenIterator<wchar_t>;
    #endif

    // Exceptions

    class SyntaxError:
    public std::runtime_error {
    public:
        SyntaxError(const u8string& text, size_t offset, const u8string& message = "Syntax error"):
            std::runtime_error(assemble(text, offset, message)),
            bug(std::make_shared<u8string>(text)), ofs(offset) {}
        const char* text() const noexcept { return bug->data(); }
        size_t offset() const noexcept { return ofs; }
    private:
        std::shared_ptr<u8string> bug;
        size_t ofs;
        static u8string assemble(const u8string& text, size_t offset, const u8string& message);
    };

    inline u8string SyntaxError::assemble(const u8string& text, size_t offset, const u8string& message) {
        u8string s = message;
        s += " at offset ";
        s += Crow::dec(offset);
        s += ": Unexpected ";
        if (text.empty()) {
            s += "EOF";
        } else {
            // TODO - smarter quoting
            s += '\"';
            s += text;
            s += '\"';
        }
        return s;
    }

    // Token structure

    template <typename C>
    struct BasicToken {
        using char_type = C;
        using string_type = basic_string<C>;
        const string_type* text;
        size_t offset;
        size_t count;
        int tag;
        operator string_type() const { return text ? text->substr(offset, count) : string_type(); }
    };

    // Token iterator

    template <typename CX>
    class BasicTokenIterator {
    public:
        using char_type = typename UnicornDetail::CharType<CX>::type;
        using string_type = basic_string<char_type>;
        using difference_type = ptrdiff_t;<commit_msg>Use quote() in lexer errors<commit_after>#pragma once

#include "unicorn/core.hpp"
#include "unicorn/character.hpp"
#include "unicorn/regex.hpp"
#include "unicorn/string.hpp"
#include <cstring>
#include <functional>
#include <memory>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <vector>

namespace Unicorn {

    // Forward declarations

    template <typename CX> class BasicLexer;
    template <typename CX> struct BasicToken;
    template <typename CX> class BasicTokenIterator;
    class SyntaxError;

    using Lexer = BasicLexer<char>;
    using Token = BasicToken<char>;
    using TokenIterator = BasicTokenIterator<char>;
    using ByteLexer = BasicLexer<ByteMode>;
    using ByteTokenIterator = BasicTokenIterator<ByteMode>;

    #if defined(UNICORN_PCRE16)
        using Lexer16 = BasicLexer<char16_t>;
        using Token16 = BasicToken<char16_t>;
        using TokenIterator16 = BasicTokenIterator<char16_t>;
    #endif

    #if defined(UNICORN_PCRE32)
        using Lexer32 = BasicLexer<char32_t>;
        using Token32 = BasicToken<char32_t>;
        using TokenIterator32 = BasicTokenIterator<char32_t>;
    #endif

    #if defined(UNICORN_PCRE_WCHAR)
        using WideLexer = BasicLexer<wchar_t>;
        using WideToken = BasicToken<wchar_t>;
        using WideTokenIterator = BasicTokenIterator<wchar_t>;
    #endif

    // Exceptions

    class SyntaxError:
    public std::runtime_error {
    public:
        SyntaxError(const u8string& text, size_t offset, const u8string& message = "Syntax error"):
            std::runtime_error(assemble(text, offset, message)),
            bug(std::make_shared<u8string>(text)), ofs(offset) {}
        const char* text() const noexcept { return bug->data(); }
        size_t offset() const noexcept { return ofs; }
    private:
        std::shared_ptr<u8string> bug;
        size_t ofs;
        static u8string assemble(const u8string& text, size_t offset, const u8string& message);
    };

    inline u8string SyntaxError::assemble(const u8string& text, size_t offset, const u8string& message) {
        u8string s = message;
        s += " at offset ";
        s += Crow::dec(offset);
        s += ": Unexpected ";
        if (text.empty())
            s += "EOF";
        else
            s += Crow::quote(text, true);
        return s;
    }

    // Token structure

    template <typename C>
    struct BasicToken {
        using char_type = C;
        using string_type = basic_string<C>;
        const string_type* text;
        size_t offset;
        size_t count;
        int tag;
        operator string_type() const { return text ? text->substr(offset, count) : string_type(); }
    };

    // Token iterator

    template <typename CX>
    class BasicTokenIterator {
    public:
        using char_type = typename UnicornDetail::CharType<CX>::type;
        using string_type = basic_string<char_type>;
        using difference_type = ptrdiff_t;
{'fraction_non_alphanumeric': 0.10283985884725257, 'fraction_numerical': 0.013107040833473365, 'mean_word_length': 2.84993531694696, 'pattern_counts': {'":': 2, '<': 65, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5066101', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1758, 'n_words': 453}