<commit_before>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "indexfile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QDebug>
#include <QtCore/QStringList>
#include <QtCore/QFile>
#include <QtCore/QtEndian>

using namespace MulaPluginStarDict;

class IndexFile::Private
{
    public:
        Private()
        {
        }

        ~Private()
        {
        }

        QList<WordEntry> wordEntryList;
};

IndexFile::IndexFile()
    : d(new Private)
{
}

IndexFile::~IndexFile()
{
}

bool
IndexFile::load(const QString& filePath, qulonglong fileSize)
{
    QFile file(filePath);
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug() << Q_FUNC_INFO << "Failed to open file:" << filePath;
        return false;
    }

    QByteArray indexDataBuffer = file.read(fileSize);

    file.close();

    qulonglong position = 0;
    d->wordEntryList.clear();
    int squint32 = sizeof(quint32);

    while (fileSize > position)
    {
        WordEntry wordEntry;
        wordEntry.setData(indexDataBuffer.mid(position));
        ++position;
        wordEntry.setDataOffset(qFromBigEndian(*reinterpret_cast<quint32 *>(indexDataBuffer.mid(position).data())));
        position += squint32;
        wordEntry.setDataSize(qFromBigEndian(*reinterpret_cast<quint32 *>(indexDataBuffer.mid(position).data())));
        position += squint32;

        d->wordEntryList.append(wordEntry);
    }

    return true;<commit_msg>Refactor the index file parsing into a forever loop and on the fly parsing<commit_after>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "indexfile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QDebug>
#include <QtCore/QStringList>
#include <QtCore/QFile>
#include <QtCore/QtEndian>

using namespace MulaPluginStarDict;

class IndexFile::Private
{
    public:
        Private()
        {
        }

        ~Private()
        {
        }

        QList<WordEntry> wordEntryList;
};

IndexFile::IndexFile()
    : d(new Private)
{
}

IndexFile::~IndexFile()
{
}

bool
IndexFile::load(const QString& filePath, qulonglong fileSize)
{
    QFile file(filePath);
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug() << Q_FUNC_INFO << "Failed to open file:" << filePath;
        return false;
    }

    d->wordEntryList.clear();
    char ch;
    QByteArray data;

    while (!file.atEnd()) {
        WordEntry wordEntry;

        data.clear();

        forever {
            if (file.read(&ch, 1) != -1) {
                if (ch) {
                    data.append(ch);
                } else {
                    break;
                }
            } else {
                return false;
            }
        }

        wordEntry.setData(data);

        wordEntry.setDataOffset(qFromBigEndian(*reinterpret_cast<quint32 *>(file.read(sizeof(quint32)).data())));
        wordEntry.setDataSize(qFromBigEndian(*reinterpret_cast<quint32 *>(file.read(sizeof(quint32)).data())));

        d->wordEntryList.append(wordEntry);
    }

    file.close();

    return true;
{'fraction_non_alphanumeric': 0.13459516298633017, 'fraction_numerical': 0.011987381703470032, 'mean_word_length': 3.0305084745762714, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1857441', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1344, 'n_words': 427}