<commit_before>#include "qmimemagicrule.h"

#include <QtCore/QString>
#include <QtCore/QList>
#include <QtCore/QStringList>
#include <QtCore/QDebug>
#include <qendian.h>

QT_BEGIN_NAMESPACE

static const char *kStrings[] = { "unknown", "string", "byte", "big16", "big32",
                                   "little16", "little32", "host16", "host32"};

struct QMimeMagicRulePrivate
{
    QMimeMagicRule::Type type;
    QString value;
    int startPos;
    int endPos;

    QByteArray pattern; // String
    quint16 value16; // *16
    quint32 value32; // *32

    typedef bool (*MatchFunction)(QMimeMagicRulePrivate* d, const QByteArray &data);

    MatchFunction matchFunction;
};

static bool matchBytes(QMimeMagicRulePrivate* d, const QByteArray& data)
{
    // Quick check
    if ((d->startPos + d->pattern.size()) > data.size())
        return false;
    // Most common: some string at position 0:
    if (d->startPos == 0 && d->startPos == d->endPos)
        return data.startsWith(d->pattern);
    // Range
    if (data.size() <= d->startPos)
        return false;

    int end = d->endPos - d->startPos + d->pattern.size();
    return QByteArray::fromRawData(data.constData() + d->startPos,
                                   qMin(end, data.size())).contains(d->pattern);
}

static bool match16(QMimeMagicRulePrivate *d, const QByteArray &data)
{
    const char *p = data.constData() + d->startPos;
    const char *e = data.constData() + qMin(data.size() - 2, d->endPos + 1);
    while (p < e) {
        if (*reinterpret_cast<const quint16*>(p) == d->value16)
            return true;
        ++p;
    }
    return false;
}

static bool match32(QMimeMagicRulePrivate *d, const QByteArray &data)
{
    const char *p = data.constData() + d->startPos;
    const char *e = data.constData() + qMin(data.size() - 4, d->endPos + 1);
    while (p < e) {
        if (*reinterpret_cast<const quint32*>(p) == d->value32)
            return true;
        ++p;
    }
    return false;
}

static bool matchInvalid(QMimeMagicRulePrivate *, const QByteArray &)
{
    return false;
}

QByteArray bytesFromSequence(const QString &sequence)
{
    QByteArray result;

    bool ok;
    if (!sequence.startsWith(QLatin1Char('\\'))) {
        char c = sequence.toInt(&ok);
        if (ok)
            result += c;
        return result;
    }

    // Expect hex format value like this: \0x7f\0x45\0x4c\0x46
    foreach (const QString &byte, sequence.split(QLatin1Char('\\'))) {
        result += char(byte.toInt(&ok, 16));
        if (!ok)
            return QByteArray();
    }

    return result;
}
<commit_msg>Oops2<commit_after>#include "qmimemagicrule.h"

#include <QtCore/QString>
#include <QtCore/QList>
#include <QtCore/QStringList>
#include <QtCore/QDebug>
#include <qendian.h>

QT_BEGIN_NAMESPACE

static const char *kStrings[] = { "unknown", "string", "byte", "big16", "big32",
                                   "little16", "little32", "host16", "host32"};

struct QMimeMagicRulePrivate
{
    QMimeMagicRule::Type type;
    QString value;
    int startPos;
    int endPos;

    QByteArray pattern; // String
    quint16 value16; // *16
    quint32 value32; // *32

    typedef bool (*MatchFunction)(QMimeMagicRulePrivate* d, const QByteArray &data);

    MatchFunction matchFunction;
};

static bool matchBytes(QMimeMagicRulePrivate* d, const QByteArray& data)
{
    // Quick check
    if ((d->startPos + d->pattern.size()) > data.size())
        return false;
    // Most common: some string at position 0:
    if (d->startPos == 0 && d->startPos == d->endPos)
        return data.startsWith(d->pattern);
    // Range
    if (data.size() <= d->startPos)
        return false;

    int end = d->endPos - d->startPos + d->pattern.size();
    return QByteArray::fromRawData(data.constData() + d->startPos,
                                   qMin(end, data.size())).contains(d->pattern);
}

static bool match16(QMimeMagicRulePrivate *d, const QByteArray &data)
{
    const char *p = data.constData() + d->startPos;
    const char *e = data.constData() + qMin(data.size() - 2, d->endPos + 1);
    while (p < e) {
        if (*reinterpret_cast<const quint16*>(p) == d->value16)
            return true;
        ++p;
    }
    return false;
}

static bool match32(QMimeMagicRulePrivate *d, const QByteArray &data)
{
    const char *p = data.constData() + d->startPos;
    const char *e = data.constData() + qMin(data.size() - 4, d->endPos + 1);
    while (p < e) {
        if (*reinterpret_cast<const quint32*>(p) == d->value32)
            return true;
        ++p;
    }
    return false;
}

static bool matchInvalid(QMimeMagicRulePrivate *, const QByteArray &)
{
    return false;
}

QByteArray bytesFromSequence(const QString &sequence)
{
    QByteArray result;

    bool ok;
    if (!sequence.startsWith(QLatin1Char('\\'))) {
        char c = sequence.toInt(&ok, 0); // autodetect
        if (ok)
            result += c;
        return result;
    }

    // Expect hex format value like this: \0x7f\0x45\0x4c\0x46
    foreach (const QString &byte, sequence.split(QLatin1Char('\\'))) {
        result += char(byte.toInt(&ok, 16));
        if (!ok)
            return QByteArray();
    }

    return result;
}

{'fraction_non_alphanumeric': 0.145821325648415, 'fraction_numerical': 0.02190201729106628, 'mean_word_length': 2.645658263305322, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6401057', 'n_tokens_mistral': 1833, 'n_tokens_neox': 1734, 'n_words': 422}