<commit_before>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2014 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.thunderbirdparser;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.CharConversionException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.james.mime4j.dom.BinaryBody;
import org.apache.james.mime4j.dom.Body;
import org.apache.james.mime4j.dom.Entity;
import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.dom.Multipart;
import org.apache.james.mime4j.dom.TextBody;
import org.apache.james.mime4j.dom.address.AddressList;
import org.apache.james.mime4j.dom.address.Mailbox;
import org.apache.james.mime4j.dom.address.MailboxList;
import org.apache.james.mime4j.dom.field.ContentDispositionField;
import org.apache.james.mime4j.dom.field.ContentTypeField;
import org.apache.james.mime4j.mboxiterator.CharBufferWrapper;
import org.apache.james.mime4j.mboxiterator.MboxIterator;
import org.apache.james.mime4j.message.DefaultMessageBuilder;
import org.apache.james.mime4j.stream.MimeConfig;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.ingest.IngestServices;

/**
 * A parser that extracts information about email messages and attachments from
 * a mbox file.
 * 
 * @author jwallace
 */
 class MboxParser {
    private static final Logger logger = Logger.getLogger(MboxParser.class.getName());
    private DefaultMessageBuilder messageBuilder;
    private IngestServices services;
    private StringBuilder errors;
    
    /**
     * The mime type string for html text.
     */
    private static final String HTML_TYPE = "text/html"; //NON-NLS
    
    /**
     * The local path of the mbox file.
     */
    private String localPath;
    
    MboxParser(IngestServices services, String localPath) {
        this.services = services;
        this.localPath = localPath;
        messageBuilder = new DefaultMessageBuilder();
        MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).build();
        // disable line length checks.
        messageBuilder.setMimeEntityConfig(config);
        errors = new StringBuilder();
    }
    
    static boolean isValidMimeTypeMbox(byte[] buffer) {
        return (new String(buffer)).startsWith("From "); //NON-NLS
    }
    
    /**
     * Parse the mbox file and get the email messages.
     * @param mboxFile
     * @return a list of the email messages in the mbox file.
     */
    List<EmailMessage> parse(File mboxFile) {
        // Detect possible charsets
        List<CharsetEncoder> encoders = getPossibleEncoders(mboxFile);
        
        CharsetEncoder theEncoder = null;
        Iterable<CharBufferWrapper> mboxIterator = null;
        // Loop through the possible encoders and find the first one that works.
        // That will usually be one of the first ones.
        for (CharsetEncoder encoder : encoders) {
            try {
                mboxIterator = MboxIterator.fromFile(mboxFile).charset(encoder.charset()).build();
                theEncoder = encoder;
                break;
            } catch (CharConversionException | UnsupportedCharsetException ex) {
                // Not the right encoder
            } catch (IllegalArgumentException ex) {
                // Not the right encoder
            } catch (IOException ex) {
                logger.log(Level.WARNING, "couldn't find mbox file.", ex); //NON-NLS
                addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToReadFile"));
                return new ArrayList<>();
            }
        }
        
        // If no encoders work, post an error message and return.
        if (mboxIterator == null || theEncoder == null) {
            addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.couldntFindCharset"));
            return new ArrayList<>();
        }
        
        List<EmailMessage> emails = new ArrayList<>();
        long failCount = 0;
        
        // Parse each message and extract an EmailMessage structure
        for (CharBufferWrapper message : mboxIterator) {
            try {
                Message msg = messageBuilder.parseMessage(message.asInputStream(theEncoder.charset()));
                emails.add(extractEmail(msg));
            } catch (IOException ex) {
                logger.log(Level.WARNING, "Failed to get message from mbox: {0}", ex.getMessage()); //NON-NLS
                failCount++;
            }
        }
        
        if (failCount > 0) {
            addErrorMessage(
                    NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToParseNMsgs", failCount));
        }
        return emails;
    }
    
    String getErrors() {
        return errors.toString();
    }
    
    /**
     * Use the information stored in the given mime4j message to populate an
     * EmailMessage.
     * 
     * @param msg
     * @return 
     */
    private EmailMessage extractEmail(Message msg) {
        EmailMessage email = new EmailMessage();
        // Basic Info
        email.setSender(getAddresses(msg.getFrom()));<commit_msg>NPE and other runtime exceptions possibly thrown handled<commit_after>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2014 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.thunderbirdparser;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.CharConversionException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.james.mime4j.dom.BinaryBody;
import org.apache.james.mime4j.dom.Body;
import org.apache.james.mime4j.dom.Entity;
import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.dom.Multipart;
import org.apache.james.mime4j.dom.TextBody;
import org.apache.james.mime4j.dom.address.AddressList;
import org.apache.james.mime4j.dom.address.Mailbox;
import org.apache.james.mime4j.dom.address.MailboxList;
import org.apache.james.mime4j.dom.field.ContentDispositionField;
import org.apache.james.mime4j.dom.field.ContentTypeField;
import org.apache.james.mime4j.mboxiterator.CharBufferWrapper;
import org.apache.james.mime4j.mboxiterator.MboxIterator;
import org.apache.james.mime4j.message.DefaultMessageBuilder;
import org.apache.james.mime4j.stream.MimeConfig;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.ingest.IngestServices;

/**
 * A parser that extracts information about email messages and attachments from
 * a mbox file.
 * 
 * @author jwallace
 */
 class MboxParser {
    private static final Logger logger = Logger.getLogger(MboxParser.class.getName());
    private DefaultMessageBuilder messageBuilder;
    private IngestServices services;
    private StringBuilder errors;
    
    /**
     * The mime type string for html text.
     */
    private static final String HTML_TYPE = "text/html"; //NON-NLS
    
    /**
     * The local path of the mbox file.
     */
    private String localPath;
    
    MboxParser(IngestServices services, String localPath) {
        this.services = services;
        this.localPath = localPath;
        messageBuilder = new DefaultMessageBuilder();
        MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).build();
        // disable line length checks.
        messageBuilder.setMimeEntityConfig(config);
        errors = new StringBuilder();
    }
    
    static boolean isValidMimeTypeMbox(byte[] buffer) {
        return (new String(buffer)).startsWith("From "); //NON-NLS
    }
    
    /**
     * Parse the mbox file and get the email messages.
     * @param mboxFile
     * @return a list of the email messages in the mbox file.
     */
    List<EmailMessage> parse(File mboxFile) {
        // Detect possible charsets
        List<CharsetEncoder> encoders = getPossibleEncoders(mboxFile);
        
        CharsetEncoder theEncoder = null;
        Iterable<CharBufferWrapper> mboxIterator = null;
        // Loop through the possible encoders and find the first one that works.
        // That will usually be one of the first ones.
        for (CharsetEncoder encoder : encoders) {
            try {
                mboxIterator = MboxIterator.fromFile(mboxFile).charset(encoder.charset()).build();
                theEncoder = encoder;
                break;
            } catch (CharConversionException | UnsupportedCharsetException ex) {
                // Not the right encoder
            } catch (IllegalArgumentException ex) {
                // Not the right encoder
            } catch (IOException ex) {
                logger.log(Level.WARNING, "couldn't find mbox file.", ex); //NON-NLS
                addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToReadFile"));
                return new ArrayList<>();
            }
        }
        
        // If no encoders work, post an error message and return.
        if (mboxIterator == null || theEncoder == null) {
            addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.couldntFindCharset"));
            return new ArrayList<>();
        }
        
        List<EmailMessage> emails = new ArrayList<>();
        long failCount = 0;
        
        // Parse each message and extract an EmailMessage structure
        for (CharBufferWrapper message : mboxIterator) {
            try {
                Message msg = messageBuilder.parseMessage(message.asInputStream(theEncoder.charset()));
                emails.add(extractEmail(msg));
            } catch (RuntimeException | IOException ex) {
                logger.log(Level.WARNING, "Failed to get message from mbox: {0}", ex.getMessage()); //NON-NLS
                failCount++;
            }
        }
        
        if (failCount > 0) {
            addErrorMessage(
                    NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToParseNMsgs", failCount));
        }
        return emails;
    }
    
    String getErrors() {
        return errors.toString();
    }
    
    /**
     * Use the information stored in the given mime4j message to populate an
     * EmailMessage.
     * 
     * @param msg
     * @return 
     */
    private EmailMessage extractEmail(Message msg) {
        EmailMessage email = new EmailMessage();
        // Basic Info
        email.setSender(getAddresses(msg.getFrom()));
{'fraction_non_alphanumeric': 0.09623858986465218, 'fraction_numerical': 0.005036197670758577, 'mean_word_length': 2.8119376124775046, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6650369', 'n_tokens_mistral': 3614, 'n_tokens_neox': 3501, 'n_words': 915}