<commit_before>/****************************************************************************
**
** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL21$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <qplatformdefs.h> // always first

#include "mimedatabase.h"

#include "mimedatabase_p.h"

#include "mimemagicrule_p.h"
#include "mimeprovider_p.h"
#include "mimetype_p.h"

#include <QtCore/QCoreApplication>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QSet>
#include <QtCore/QBuffer>
#include <QtCore/QUrl>
#include <QtCore/QStack>
#include <QtCore/QDebug>

#include <algorithm>
#include <functional>

static const char ALL_FILES_FILTER[]      = QT_TRANSLATE_NOOP("Core", "All Files (*)");

using namespace Utils;
using namespace Utils::Internal;

Q_GLOBAL_STATIC(MimeDatabasePrivate, staticMimeDatabase)

MimeDatabasePrivate *MimeDatabasePrivate::instance()
{
    return staticMimeDatabase();
}

MimeDatabasePrivate::MimeDatabasePrivate()
    : m_provider(0), m_defaultMimeType(QLatin1String("application/octet-stream"))
{
}

MimeDatabasePrivate::~MimeDatabasePrivate()
{
    delete m_provider;
    m_provider = 0;
}

MimeProviderBase *MimeDatabasePrivate::provider()
{
    if (!m_provider) {
//        MimeProviderBase *binaryProvider = new MimeBinaryProvider(this);
//        if (binaryProvider->isValid()) {
//            m_provider = binaryProvider;
//        } else {
//            delete binaryProvider;
        m_provider = new MimeXMLProvider(this);
//        }
    }
    return m_provider;
}

void MimeDatabasePrivate::setProvider(MimeProviderBase *theProvider)
{
    delete m_provider;
    m_provider = theProvider;
}

/*!
    \internal
    Returns a MIME type or an invalid one if none found
 */
MimeType MimeDatabasePrivate::mimeTypeForName(const QString &nameOrAlias)
{
    return provider()->mimeTypeForName(provider()->resolveAlias(nameOrAlias));
}

QStringList MimeDatabasePrivate::mimeTypeForFileName(const QString &fileName, QString *foundSuffix)
{
    if (fileName.endsWith(QLatin1Char('/')))
        return QStringList() << QLatin1String("inode/directory");

    const QStringList matchingMimeTypes = provider()->findByFileName(QFileInfo(fileName).fileName(), foundSuffix);
    return matchingMimeTypes;
}

static inline bool isTextFile(const QByteArray &data)
{
    // UTF16 byte order marks
    static const char bigEndianBOM[] = "\xFE\xFF";
    static const char littleEndianBOM[] = "\xFF\xFE";
    if (data.startsWith(bigEndianBOM) || data.startsWith(littleEndianBOM))
        return true;

    // Check the first 32 bytes (see shared-mime spec)
    const char *p = data.constData();
    const char *e = p + qMin(32, data.size());
    for ( ; p < e; ++p) {
        if ((unsigned char)(*p) < 32 && *p != 9 && *p !=10 && *p != 13)
            return false;
    }

    return true;
}

MimeType MimeDatabasePrivate::findByData(const QByteArray &data, int *accuracyPtr)
{
    if (data.isEmpty()) {
        *accuracyPtr = 100;
        return mimeTypeForName(QLatin1String("application/x-zerosize"));
    }

    *accuracyPtr = 0;
    MimeType candidate = provider()->findByMagic(data, accuracyPtr);

    if (candidate.isValid())
        return candidate;

    if (isTextFile(data)) {
        *accuracyPtr = 5;
        return mimeTypeForName(QLatin1String("text/plain"));
    }

    return mimeTypeForName(defaultMimeType());
}

MimeType MimeDatabasePrivate::mimeTypeForFileNameAndData(const QString &fileName, QIODevice *device, int *accuracyPtr)
{
    // First, glob patterns are evaluated. If there is a match with max weight,
    // this one is selected and we are done. Otherwise, the file contents are
    // evaluated and the match with the highest value (either a magic priority or
    // a glob pattern weight) is selected. Matching starts from max level (most
    // specific) in both cases, even when there is already a suffix matching candidate.
    *accuracyPtr = 0;

    // Pass 1) Try to match on the file name
    QStringList candidatesByName = mimeTypeForFileName(fileName);
    if (candidatesByName.count() == 1) {
        *accuracyPtr = 100;
        const MimeType mime = mimeTypeForName(candidatesByName.at(0));
        if (mime.isValid())<commit_msg>MimeData: Never detect anything as application/x-zerosize<commit_after>/****************************************************************************
**
** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL21$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <qplatformdefs.h> // always first

#include "mimedatabase.h"

#include "mimedatabase_p.h"

#include "mimemagicrule_p.h"
#include "mimeprovider_p.h"
#include "mimetype_p.h"

#include <QtCore/QCoreApplication>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QSet>
#include <QtCore/QBuffer>
#include <QtCore/QUrl>
#include <QtCore/QStack>
#include <QtCore/QDebug>

#include <algorithm>
#include <functional>

static const char ALL_FILES_FILTER[]      = QT_TRANSLATE_NOOP("Core", "All Files (*)");

using namespace Utils;
using namespace Utils::Internal;

Q_GLOBAL_STATIC(MimeDatabasePrivate, staticMimeDatabase)

MimeDatabasePrivate *MimeDatabasePrivate::instance()
{
    return staticMimeDatabase();
}

MimeDatabasePrivate::MimeDatabasePrivate()
    : m_provider(0), m_defaultMimeType(QLatin1String("application/octet-stream"))
{
}

MimeDatabasePrivate::~MimeDatabasePrivate()
{
    delete m_provider;
    m_provider = 0;
}

MimeProviderBase *MimeDatabasePrivate::provider()
{
    if (!m_provider) {
//        MimeProviderBase *binaryProvider = new MimeBinaryProvider(this);
//        if (binaryProvider->isValid()) {
//            m_provider = binaryProvider;
//        } else {
//            delete binaryProvider;
        m_provider = new MimeXMLProvider(this);
//        }
    }
    return m_provider;
}

void MimeDatabasePrivate::setProvider(MimeProviderBase *theProvider)
{
    delete m_provider;
    m_provider = theProvider;
}

/*!
    \internal
    Returns a MIME type or an invalid one if none found
 */
MimeType MimeDatabasePrivate::mimeTypeForName(const QString &nameOrAlias)
{
    return provider()->mimeTypeForName(provider()->resolveAlias(nameOrAlias));
}

QStringList MimeDatabasePrivate::mimeTypeForFileName(const QString &fileName, QString *foundSuffix)
{
    if (fileName.endsWith(QLatin1Char('/')))
        return QStringList() << QLatin1String("inode/directory");

    const QStringList matchingMimeTypes = provider()->findByFileName(QFileInfo(fileName).fileName(), foundSuffix);
    return matchingMimeTypes;
}

static inline bool isTextFile(const QByteArray &data)
{
    // UTF16 byte order marks
    static const char bigEndianBOM[] = "\xFE\xFF";
    static const char littleEndianBOM[] = "\xFF\xFE";
    if (data.startsWith(bigEndianBOM) || data.startsWith(littleEndianBOM))
        return true;

    // Check the first 32 bytes (see shared-mime spec)
    const char *p = data.constData();
    const char *e = p + qMin(32, data.size());
    for ( ; p < e; ++p) {
        if ((unsigned char)(*p) < 32 && *p != 9 && *p !=10 && *p != 13)
            return false;
    }

    return true;
}

MimeType MimeDatabasePrivate::findByData(const QByteArray &data, int *accuracyPtr)
{
//    if (data.isEmpty()) {
//        *accuracyPtr = 100;
//        return mimeTypeForName(QLatin1String("application/x-zerosize"));
//    }

    *accuracyPtr = 0;
    MimeType candidate = provider()->findByMagic(data, accuracyPtr);

    if (candidate.isValid())
        return candidate;

    if (isTextFile(data)) {
        *accuracyPtr = 5;
        return mimeTypeForName(QLatin1String("text/plain"));
    }

    return mimeTypeForName(defaultMimeType());
}

MimeType MimeDatabasePrivate::mimeTypeForFileNameAndData(const QString &fileName, QIODevice *device, int *accuracyPtr)
{
    // First, glob patterns are evaluated. If there is a match with max weight,
    // this one is selected and we are done. Otherwise, the file contents are
    // evaluated and the match with the highest value (either a magic priority or
    // a glob pattern weight) is selected. Matching starts from max level (most
    // specific) in both cases, even when there is already a suffix matching candidate.
    *accuracyPtr = 0;

    // Pass 1) Try to match on the file name
    QStringList candidatesByName = mimeTypeForFileName(fileName);
    if (candidatesByName.count() == 1) {
        *accuracyPtr = 100;
        const MimeType mime = mimeTypeForName(candidatesByName.at(0));
        if (mime.isValid())
{'fraction_non_alphanumeric': 0.13562466866937622, 'fraction_numerical': 0.0084820639688991, 'mean_word_length': 4.117088607594937, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2754152', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3302, 'n_words': 992}