<commit_before>/***************************************************************************
                          filter_mbox.cxx  -  mbox mail import
                             -------------------
    begin                : Sat Apr 5 2003
    copyright            : (C) 2003 by <NAME>
    email                : <EMAIL>
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#include <config.h>
#include <klocale.h>
#include <kfiledialog.h>
#include <ktempfile.h>
#include <kdebug.h>

#include "filter_mbox.hxx"


FilterMBox::FilterMBox() :
        Filter( i18n("Import mbox Files (UNIX, Evolution)"),
                "<NAME> <p>( Filter accelerated by <NAME> )</p>",
                i18n("<p><b>mbox import filter</b></p>"
                     "<p>This filter will import mbox files into KMail. Use this filter "
                     "if you want to import mails from Ximian Evolution or other mailers "
                     "that use this traditional UNIX format.</p>"
                     "<p><b>Note:</b> Emails will be imported into folders named after the "
                     "file they came from, prefixed with MBOX-</p>" ))
{}

FilterMBox::~FilterMBox()
{
    endImport();
}

void FilterMBox::import(FilterInfo *info)
{
    int currentFile = 1;
    int overall_status = 0;
    bool first_msg = true;

    QStringList filenames = KFileDialog::getOpenFileNames( QDir::homeDirPath(), "*|" + i18n("mbox Files (*)"), info->parent() );
    info->setOverall(0);

    for ( QStringList::Iterator filename = filenames.begin(); filename != filenames.end(); ++filename, ++currentFile) {
        QFile mbox( *filename );
        if (! mbox.open( IO_ReadOnly ) ) {
            info->alert( i18n("Unable to open %1, skipping").arg( *filename ) );
        } else {
            QFileInfo filenameInfo( *filename );
            QString folderName( "MBOX-" + filenameInfo.baseName(TRUE) );

            info->setCurrent(0);
            info->addLog( i18n("Importing emails from %1...").arg( *filename ) );

            info->setFrom( *filename );
            info->setTo( folderName );

            QByteArray input(MAX_LINE);
            long l = 0;

            while ( ! mbox.atEnd() ) {
                KTempFile tmp;
                QIODevice::Offset filepos = 0;
                /* comment by Danny:
                * Don't use QTextStream to read from mbox, better use QDataStream. QTextStream only 
                * support Unicode/Latin1/Locale. So you lost information from emails with 
                * charset!=Unicode/Latin1/Locale (e.g. KOI8-R) and Content-Transfer-Encoding != base64 
                * (e.g. 8Bit). It also not help to convert the QTextStream to Unicode. By this you
                * get Unicode/UTF-email but KMail can't detect the correct charset.
                */
                QCString seperate;

                if(!first_msg)
                    tmp.file()->writeBlock( input, l );
                l = mbox.readLine( input.data(),MAX_LINE); // read the first line, prevent "From "
                tmp.file()->writeBlock( input, l );

                while ( ! mbox.atEnd() &&  (l = mbox.readLine(input.data(),MAX_LINE)) && ((seperate = input.data()).left(5) != "From ")) {
                       tmp.file()->writeBlock( input, l );
                       
                       // workaround to fix hang if a corrupted mbox contains some 
                       // binary data, for more see bug #106796
                       if (mbox.at() == filepos)
                           mbox.at(mbox.size());
                       else 
                           filepos = mbox.at();
                }<commit_msg>* fixed import of mboxes from maillinglist archives which contain a    line in the mailheader starting with "From " (instead of "From: ").   This should allow to move imported mails to IMAP servers (Cyrus    e.g. don't accept mail with such a header line)<commit_after>/***************************************************************************
                          filter_mbox.cxx  -  mbox mail import
                             -------------------
    begin                : Sat Apr 5 2003
    copyright            : (C) 2003 by <NAME>
    email                : <EMAIL>
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#include <config.h>
#include <klocale.h>
#include <kfiledialog.h>
#include <ktempfile.h>
#include <kdebug.h>

#include "filter_mbox.hxx"


FilterMBox::FilterMBox() :
        Filter( i18n("Import mbox Files (UNIX, Evolution)"),
                "<NAME> <p>( Filter accelerated by <NAME> )</p>",
                i18n("<p><b>mbox import filter</b></p>"
                     "<p>This filter will import mbox files into KMail. Use this filter "
                     "if you want to import mails from Ximian Evolution or other mailers "
                     "that use this traditional UNIX format.</p>"
                     "<p><b>Note:</b> Emails will be imported into folders named after the "
                     "file they came from, prefixed with MBOX-</p>" ))
{}

FilterMBox::~FilterMBox()
{
    endImport();
}

void FilterMBox::import(FilterInfo *info)
{
    int currentFile = 1;
    int overall_status = 0;
    bool first_msg = true;

    QStringList filenames = KFileDialog::getOpenFileNames( QDir::homeDirPath(), "*|" + i18n("mbox Files (*)"), info->parent() );
    info->setOverall(0);

    for ( QStringList::Iterator filename = filenames.begin(); filename != filenames.end(); ++filename, ++currentFile) {
        QFile mbox( *filename );
        if (! mbox.open( IO_ReadOnly ) ) {
            info->alert( i18n("Unable to open %1, skipping").arg( *filename ) );
        } else {
            QFileInfo filenameInfo( *filename );
            QString folderName( "MBOX-" + filenameInfo.baseName(TRUE) );

            info->setCurrent(0);
            info->addLog( i18n("Importing emails from %1...").arg( *filename ) );

            info->setFrom( *filename );
            info->setTo( folderName );

            QByteArray input(MAX_LINE);
            long l = 0;

            while ( ! mbox.atEnd() ) {
                KTempFile tmp;
                QIODevice::Offset filepos = 0;
                /* comment by Danny:
                * Don't use QTextStream to read from mbox, better use QDataStream. QTextStream only 
                * support Unicode/Latin1/Locale. So you lost information from emails with 
                * charset!=Unicode/Latin1/Locale (e.g. KOI8-R) and Content-Transfer-Encoding != base64 
                * (e.g. 8Bit). It also not help to convert the QTextStream to Unicode. By this you
                * get Unicode/UTF-email but KMail can't detect the correct charset.
                */
                QCString seperate;

		/* check if the first line start with "From " (and not "From: ") and discard the line 
		 * in this case because some IMAP servers (e.g. Cyrus) don't accept this header line */
                if(!first_msg && ((seperate = input.data()).left(5) != "From "))
                    tmp.file()->writeBlock( input, l );

                l = mbox.readLine( input.data(),MAX_LINE); // read the first line, prevent "From "

		if ((seperate = input.data()).left(5) != "From ")
	                tmp.file()->writeBlock( input, l );

                while ( ! mbox.atEnd() &&  (l = mbox.readLine(input.data(),MAX_LINE)) && ((seperate = input.data()).left(5) != "From ")) {
                       tmp.file()->writeBlock( input, l );
                       
                       // workaround to fix hang if a corrupted mbox contains some 
                       // binary data, for more see bug #106796
                       if (mbox.at() == filepos)
                           mbox.at(mbox.size());
                       else 
                           filepos = mbox.at();
                }
{'fraction_non_alphanumeric': 0.17562527424308907, 'fraction_numerical': 0.009214567792891619, 'mean_word_length': 1.7337331334332833, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '888466', 'n_tokens_mistral': 2461, 'n_tokens_neox': 2237, 'n_words': 702}