<commit_before>/*  This file is part of the Kate project.
 *
 *  Copyright (C) 2010 <NAME> <<EMAIL>>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 *  You should have received a copy of the GNU Library General Public License
 *  along with this library; see the file COPYING.LIB.  If not, write to
 *  the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA 02110-1301, USA.
 */

#ifndef KATE_TEXTLOADER_H
#define KATE_TEXTLOADER_H

#include <QtCore/QString>
#include <QtCore/QFile>
#include <QtCore/QCryptographicHash>

// on the fly compression
#include <kfilterdev.h>
#include <kmimetype.h>

namespace Kate {

/**
 * loader block size, load 256 kb at once per default
 * if file size is smaller, fall back to file size
 * must be a multiple of 2
 */
static const qint64 KATE_FILE_LOADER_BS  = 256 * 1024;

/**
 * File Loader, will handle reading of files + detecting encoding
 */
class TextLoader
{
  public:
    /**
     * Construct file loader for given file.
     * @param filename file to open
     * @param proberType prober type
     */
    TextLoader (const QString &filename, KEncodingProber::ProberType proberType)
      : m_codec (0)
      , m_eof (false) // default to not eof
      , m_lastWasEndOfLine (true) // at start of file, we had a virtual newline
      , m_lastWasR (false) // we have not found a \r as last char
      , m_position (0)
      , m_lastLineStart (0)
      , m_eol (TextBuffer::eolUnknown) // no eol type detected atm
      , m_buffer (KATE_FILE_LOADER_BS, 0)
      , m_digest (QCryptographicHash::Md5)
      , m_converterState (0)
      , m_bomFound (false)
      , m_firstRead (true)
      , m_proberType (proberType)
    {
      // try to get mimetype for on the fly decompression, don't rely on filename!
      QFile testMime (filename);
      if (testMime.open (QIODevice::ReadOnly))
        m_mimeType = KMimeType::findByContent (&testMime)->name ();
      else
        m_mimeType = KMimeType::findByPath (filename, 0, false)->name ();

      // construct filter device
      m_file = new QFile (filename); //FIXME KF5 KFilterDev::deviceForFile (filename, m_mimeType, false);
    }

    /**
     * Destructor
     */
    ~TextLoader ()
    {
      delete m_file;
      delete m_converterState;
    }

    /**
     * open file with given codec
     * @param codec codec to use, if 0, will do some auto-dectect or fallback
     * @return success
     */
    bool open (QTextCodec *codec)
    {
      m_codec = codec;
      m_eof = false;
      m_lastWasEndOfLine = true;
      m_lastWasR = false;
      m_position = 0;
      m_lastLineStart = 0;
      m_eol = TextBuffer::eolUnknown;<commit_msg>bring back loading of compressed files<commit_after>/*  This file is part of the Kate project.
 *
 *  Copyright (C) 2010 <NAME> <<EMAIL>>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 *  You should have received a copy of the GNU Library General Public License
 *  along with this library; see the file COPYING.LIB.  If not, write to
 *  the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA 02110-1301, USA.
 */

#ifndef KATE_TEXTLOADER_H
#define KATE_TEXTLOADER_H

#include <QtCore/QString>
#include <QtCore/QFile>
#include <QtCore/QCryptographicHash>

// on the fly compression
#include <kfilterdev.h>
#include <kmimetype.h>

namespace Kate {

/**
 * loader block size, load 256 kb at once per default
 * if file size is smaller, fall back to file size
 * must be a multiple of 2
 */
static const qint64 KATE_FILE_LOADER_BS  = 256 * 1024;

/**
 * File Loader, will handle reading of files + detecting encoding
 */
class TextLoader
{
  public:
    /**
     * Construct file loader for given file.
     * @param filename file to open
     * @param proberType prober type
     */
    TextLoader (const QString &filename, KEncodingProber::ProberType proberType)
      : m_codec (0)
      , m_eof (false) // default to not eof
      , m_lastWasEndOfLine (true) // at start of file, we had a virtual newline
      , m_lastWasR (false) // we have not found a \r as last char
      , m_position (0)
      , m_lastLineStart (0)
      , m_eol (TextBuffer::eolUnknown) // no eol type detected atm
      , m_buffer (KATE_FILE_LOADER_BS, 0)
      , m_digest (QCryptographicHash::Md5)
      , m_converterState (0)
      , m_bomFound (false)
      , m_firstRead (true)
      , m_proberType (proberType)
    {
      // try to get mimetype for on the fly decompression, don't rely on filename!
      QFile testMime (filename);
      if (testMime.open (QIODevice::ReadOnly))
        m_mimeType = KMimeType::findByContent (&testMime)->name ();
      else
        m_mimeType = KMimeType::findByPath (filename, 0, false)->name ();

      // construct filter device
      KCompressionDevice::CompressionType compressionType = KFilterDev::compressionTypeForMimeType(m_mimeType);
      m_file = new KCompressionDevice(filename, compressionType);
    }

    /**
     * Destructor
     */
    ~TextLoader ()
    {
      delete m_file;
      delete m_converterState;
    }

    /**
     * open file with given codec
     * @param codec codec to use, if 0, will do some auto-dectect or fallback
     * @return success
     */
    bool open (QTextCodec *codec)
    {
      m_codec = codec;
      m_eof = false;
      m_lastWasEndOfLine = true;
      m_lastWasR = false;
      m_position = 0;
      m_lastLineStart = 0;
      m_eol = TextBuffer::eolUnknown;
{'fraction_non_alphanumeric': 0.08663017982799062, 'fraction_numerical': 0.01235340109460516, 'mean_word_length': 2.8905109489051095, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2461592', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1970, 'n_words': 717}