<commit_before>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "sdfinputstream.h"
#include "subinputstream.h"
#include "kmpsearcher.h"
#include <iostream>
#include <sstream>
#include <cstring>

using namespace std;
using namespace Strigi;

const string SdfInputStream::delimiter("$$$$");
const string SdfInputStream::label("V2000");

SdfInputStream::SdfInputStream(InputStream* input)
        : SubStreamProvider(input), substream(0), entrynumber(0),
        previousStartOfDelimiter(0) {
    m_searcher.setQuery(delimiter);
}
SdfInputStream::~SdfInputStream() {
    if (substream && substream != m_entrystream) {
        delete substream;
    }
}
bool
SdfInputStream::checkHeader(const char* data, int32_t datasize) {
    // expect at least 64 bytes
    if (datasize < 64) return false;

    KmpSearcher searcher;
    searcher.setQuery(label);
    const char* found = searcher.search(data, datasize);

    return found > 0;
}
InputStream*
SdfInputStream::nextEntry() {
    if (m_status != Ok) return 0;

    m_input->reset(previousStartOfDelimiter);

    // read anything that's left over in the previous substream
    if (substream) {
        substream->reset(0);
        const char* dummy;
        while (substream->status() == Ok) {
            substream->read(dummy, 1, 0);
        }
        if (substream->status() == Error) {
            m_status = Error;
        }
        if (substream && substream != m_entrystream) {
            delete substream;
        }
        substream = 0;
        delete m_entrystream;
        m_entrystream = 0;
    
        m_input->reset(previousStartOfDelimiter);
    
        // eat delimiter and following newlines
        if (m_input->status() == Ok) {
            m_input->read(dummy, 4, 4);
            if (strncmp(dummy, delimiter.c_str(), 4) == 0) {
                m_input->read(dummy, 1, 1);
                while (m_input->status() == Ok && (
                        strncmp(dummy, "\n", 1) == 0
                        || strncmp(dummy, "\r", 1) == 0)) {
                    m_input->read(dummy, 1, 1);
                }
            }
        }
    }
    // make sure it is not a MOL
    // we can not check it in checkHeader due to low header size limit
    // There is only one way to destinguish between MOL and SD:
    // MOL does not have $$$$ delimiter. Return no entries if it is a MOL.
    const char* start;
    const char* end;
    int32_t nread = 0;
    int32_t total = 0;
    const int64_t pos = m_input->position();
    int64_t len=0;<commit_msg>Fix out of bounds reading.<commit_after>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "sdfinputstream.h"
#include "subinputstream.h"
#include "kmpsearcher.h"
#include <iostream>
#include <sstream>
#include <cstring>

using namespace std;
using namespace Strigi;

const string SdfInputStream::delimiter("$$$$");
const string SdfInputStream::label("V2000");

SdfInputStream::SdfInputStream(InputStream* input)
        : SubStreamProvider(input), substream(0), entrynumber(0),
        previousStartOfDelimiter(0) {
    m_searcher.setQuery(delimiter);
}
SdfInputStream::~SdfInputStream() {
    if (substream && substream != m_entrystream) {
        delete substream;
    }
}
bool
SdfInputStream::checkHeader(const char* data, int32_t datasize) {
    // expect at least 64 bytes
    if (datasize < 64) return false;

    KmpSearcher searcher;
    searcher.setQuery(label);
    const char* found = searcher.search(data, datasize);

    return found > 0;
}
InputStream*
SdfInputStream::nextEntry() {
    if (m_status != Ok) return 0;

    m_input->reset(previousStartOfDelimiter);
    int32_t nread = 0;

    // read anything that's left over in the previous substream
    if (substream) {
        substream->reset(0);
        const char* dummy;
        while (substream->status() == Ok) {
            substream->read(dummy, 1, 0);
        }
        if (substream->status() == Error) {
            m_status = Error;
        }
        if (substream && substream != m_entrystream) {
            delete substream;
        }
        substream = 0;
        delete m_entrystream;
        m_entrystream = 0;
    
        m_input->reset(previousStartOfDelimiter);
    
        // eat delimiter and following newlines
        if (m_input->status() == Ok) {
            nread = m_input->read(dummy, 4, 4);
            if (nread == 4 && strncmp(dummy, delimiter.c_str(), 4) == 0) {
                m_input->read(dummy, 1, 1);
                while (m_input->status() == Ok && (
                        strncmp(dummy, "\n", 1) == 0
                        || strncmp(dummy, "\r", 1) == 0)) {
                    m_input->read(dummy, 1, 1);
                }
            }
        }
    }
    // make sure it is not a MOL
    // we can not check it in checkHeader due to low header size limit
    // There is only one way to destinguish between MOL and SD:
    // MOL does not have $$$$ delimiter. Return no entries if it is a MOL.
    const char* start;
    const char* end;
    int32_t total = 0;
    const int64_t pos = m_input->position();
    int64_t len=0;
{'fraction_non_alphanumeric': 0.1061260992696378, 'fraction_numerical': 0.017737367715009687, 'mean_word_length': 2.5959271168274385, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3603926', 'n_tokens_mistral': 2066, 'n_tokens_neox': 1961, 'n_words': 688}