<commit_before>#include <qpdf/QPDF_String.hh>

#include <qpdf/QUtil.hh>

// DO NOT USE ctype -- it is locale dependent for some things, and
// it's not worth the risk of including it in case it may accidentally
// be used.
#include <string.h>

// See above about ctype.
static bool
is_ascii_printable(char ch)
{
    return ((ch >= 32) && (ch <= 126));
}
static bool
is_iso_latin1_printable(char ch)
{
    return (
        ((ch >= 32) && (ch <= 126)) || (static_cast<unsigned char>(ch) >= 160));
}

QPDF_String::QPDF_String(std::string const& val) :
    QPDFValue(::ot_string, "string"),
    val(val)
{
}

std::shared_ptr<QPDFObject>
QPDF_String::create(std::string const& val)
{
    return do_create(new QPDF_String(val));
}

std::shared_ptr<QPDFObject>
QPDF_String::create_utf16(std::string const& utf8_val)
{
    std::string result;
    if (!QUtil::utf8_to_pdf_doc(utf8_val, result, '?')) {
        result = QUtil::utf8_to_utf16(utf8_val);
    }
    return do_create(new QPDF_String(result));
}

std::shared_ptr<QPDFObject>
QPDF_String::shallowCopy()
{
    return create(val);
}

std::string
QPDF_String::unparse()
{
    return unparse(false);
}

JSON
QPDF_String::getJSON(int json_version)
{
    if (json_version == 1) {
        return JSON::makeString(getUTF8Val());
    }
    // See if we can unambiguously represent as Unicode.
    bool is_unicode = false;
    std::string result;
    std::string candidate = getUTF8Val();
    if (QUtil::is_utf16(this->val) || QUtil::is_explicit_utf8(this->val)) {
        is_unicode = true;
        result = candidate;
    } else if (!useHexString()) {
        std::string test;
        if (QUtil::utf8_to_pdf_doc(candidate, test, '?') &&
            (test == this->val)) {
            // This is a PDF-doc string that can be losslessly encoded
            // as Unicode.
            is_unicode = true;
            result = candidate;
        }
    }
    if (is_unicode) {
        result = "u:" + result;
    } else {
        result = "b:" + QUtil::hex_encode(this->val);
    }
    return JSON::makeString(result);
}

bool
QPDF_String::useHexString() const
{
    // Heuristic: use the hexadecimal representation of a string if
    // there are any non-printable (in PDF Doc encoding) characters or
    // if too large of a proportion of the string consists of
    // non-ASCII characters.
    bool nonprintable = false;
    unsigned int non_ascii = 0;
    for (unsigned int i = 0; i < this->val.length(); ++i) {
        char ch = this->val.at(i);
        if ((ch == 0) ||
            (!(is_ascii_printable(ch) || strchr("\n\r\t\b\f", ch)))) {
            if ((ch >= 0) && (ch < 24)) {
                nonprintable = true;
            }
            ++non_ascii;
        }
    }
    return (nonprintable || (5 * non_ascii > val.length()));
}

std::string
QPDF_String::unparse(bool force_binary)
{
    bool use_hexstring = force_binary || useHexString();
    std::string result;
    if (use_hexstring) {
        static auto constexpr hexchars = "0123456789abcdef";<commit_msg>Tune QPDF_String::useHexString()<commit_after>#include <qpdf/QPDF_String.hh>

#include <qpdf/QUtil.hh>

// DO NOT USE ctype -- it is locale dependent for some things, and
// it's not worth the risk of including it in case it may accidentally
// be used.
#include <string.h>

static bool
is_iso_latin1_printable(char ch)
{
    return (
        ((ch >= 32) && (ch <= 126)) || (static_cast<unsigned char>(ch) >= 160));
}

QPDF_String::QPDF_String(std::string const& val) :
    QPDFValue(::ot_string, "string"),
    val(val)
{
}

std::shared_ptr<QPDFObject>
QPDF_String::create(std::string const& val)
{
    return do_create(new QPDF_String(val));
}

std::shared_ptr<QPDFObject>
QPDF_String::create_utf16(std::string const& utf8_val)
{
    std::string result;
    if (!QUtil::utf8_to_pdf_doc(utf8_val, result, '?')) {
        result = QUtil::utf8_to_utf16(utf8_val);
    }
    return do_create(new QPDF_String(result));
}

std::shared_ptr<QPDFObject>
QPDF_String::shallowCopy()
{
    return create(val);
}

std::string
QPDF_String::unparse()
{
    return unparse(false);
}

JSON
QPDF_String::getJSON(int json_version)
{
    if (json_version == 1) {
        return JSON::makeString(getUTF8Val());
    }
    // See if we can unambiguously represent as Unicode.
    bool is_unicode = false;
    std::string result;
    std::string candidate = getUTF8Val();
    if (QUtil::is_utf16(this->val) || QUtil::is_explicit_utf8(this->val)) {
        is_unicode = true;
        result = candidate;
    } else if (!useHexString()) {
        std::string test;
        if (QUtil::utf8_to_pdf_doc(candidate, test, '?') &&
            (test == this->val)) {
            // This is a PDF-doc string that can be losslessly encoded
            // as Unicode.
            is_unicode = true;
            result = candidate;
        }
    }
    if (is_unicode) {
        result = "u:" + result;
    } else {
        result = "b:" + QUtil::hex_encode(this->val);
    }
    return JSON::makeString(result);
}

bool
QPDF_String::useHexString() const
{
    // Heuristic: use the hexadecimal representation of a string if
    // there are any non-printable (in PDF Doc encoding) characters or
    // if too large of a proportion of the string consists of
    // non-ASCII characters.
    unsigned int non_ascii = 0;
    for (auto const ch: this->val) {
        if (ch > 126) {
            ++non_ascii;
        } else if (ch >= 32) {
            continue;
        } else if (ch < 0 || ch >= 24) {
            ++non_ascii;
        } else if (!(ch == '\n' || ch == '\r' || ch == '\t' || ch == '\b' ||
                     ch == '\f')) {
            return true;
        }
    }
    return 5 * non_ascii > val.length();
}

std::string
QPDF_String::unparse(bool force_binary)
{
    bool use_hexstring = force_binary || useHexString();
    std::string result;
    if (use_hexstring) {
        static auto constexpr hexchars = "0123456789abcdef";
{'fraction_non_alphanumeric': 0.14324324324324325, 'fraction_numerical': 0.015540540540540541, 'mean_word_length': 2.7569796954314723, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4000870', 'n_tokens_mistral': 2079, 'n_tokens_neox': 1989, 'n_words': 491}