<commit_before>#include <qpdf/QPDFTokenizer.hh>

// DO NOT USE ctype -- it is locale dependent for some things, and
// it's not worth the risk of including it in case it may accidentally
// be used.

#include <qpdf/QTC.hh>
#include <qpdf/QPDFExc.hh>
#include <qpdf/QUtil.hh>
#include <qpdf/QPDFObjectHandle.hh>

#include <stdexcept>
#include <string.h>
#include <cstdlib>

QPDFTokenizer::Members::Members() :
    pound_special_in_name(true),
    allow_eof(false),
    include_ignorable(false)
{
    reset();
}

void
QPDFTokenizer::Members::reset()
{
    state = st_top;
    type = tt_bad;
    val = "";
    raw_val = "";
    error_message = "";
    unread_char = false;
    char_to_unread = '\0';
    string_depth = 0;
    string_ignoring_newline = false;
    last_char_was_bs = false;
}

QPDFTokenizer::Members::~Members()
{
}

QPDFTokenizer::Token::Token(token_type_e type, std::string const& value) :
    type(type),
    value(value),
    raw_value(value)
{
    if (type == tt_string)
    {
        raw_value = QPDFObjectHandle::newString(value).unparse();
    }
    else if (type == tt_string)
    {
        raw_value = QPDFObjectHandle::newName(value).unparse();
    }
}



QPDFTokenizer::QPDFTokenizer() :<commit_msg>Fix small logic error in Token construct (fixes #206)<commit_after>#include <qpdf/QPDFTokenizer.hh>

// DO NOT USE ctype -- it is locale dependent for some things, and
// it's not worth the risk of including it in case it may accidentally
// be used.

#include <qpdf/QTC.hh>
#include <qpdf/QPDFExc.hh>
#include <qpdf/QUtil.hh>
#include <qpdf/QPDFObjectHandle.hh>

#include <stdexcept>
#include <string.h>
#include <cstdlib>

QPDFTokenizer::Members::Members() :
    pound_special_in_name(true),
    allow_eof(false),
    include_ignorable(false)
{
    reset();
}

void
QPDFTokenizer::Members::reset()
{
    state = st_top;
    type = tt_bad;
    val = "";
    raw_val = "";
    error_message = "";
    unread_char = false;
    char_to_unread = '\0';
    string_depth = 0;
    string_ignoring_newline = false;
    last_char_was_bs = false;
}

QPDFTokenizer::Members::~Members()
{
}

QPDFTokenizer::Token::Token(token_type_e type, std::string const& value) :
    type(type),
    value(value),
    raw_value(value)
{
    if (type == tt_string)
    {
        raw_value = QPDFObjectHandle::newString(value).unparse();
    }
    else if (type == tt_name)
    {
        raw_value = QPDFObjectHandle::newName(value).unparse();
    }
}



QPDFTokenizer::QPDFTokenizer() :
{'fraction_non_alphanumeric': 0.1388888888888889, 'fraction_numerical': 0.002818035426731079, 'mean_word_length': 3.62756052141527, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7523849', 'n_tokens_mistral': 907, 'n_tokens_neox': 909, 'n_words': 184}