<commit_before>package com.fsck.k9.mail.internet;

import android.util.Log;

import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.BodyPart;
import com.fsck.k9.mail.Message;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.Multipart;
import com.fsck.k9.mail.Part;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.fsck.k9.mail.K9MailLib.LOG_TAG;
import static com.fsck.k9.mail.internet.CharsetSupport.fixupCharset;
import static com.fsck.k9.mail.internet.MimeUtility.getHeaderParameter;
import static com.fsck.k9.mail.internet.Viewable.Alternative;
import static com.fsck.k9.mail.internet.Viewable.Html;
import static com.fsck.k9.mail.internet.Viewable.MessageHeader;
import static com.fsck.k9.mail.internet.Viewable.Text;
import static com.fsck.k9.mail.internet.Viewable.Textual;

public class MessageExtractor {
    private MessageExtractor() {}

    public static String getTextFromPart(Part part) {
        try {
            if ((part != null) && (part.getBody() != null)) {
                final Body body = part.getBody();
                if (body instanceof TextBody) {
                    return ((TextBody)body).getText();
                }

                final String mimeType = part.getMimeType();
                if ((mimeType != null) && MimeUtility.mimeTypeMatches(mimeType, "text/*")) {
                    /*
                     * We've got a text part, so let's see if it needs to be processed further.
                     */
                    String charset = getHeaderParameter(part.getContentType(), "charset");
                    /*
                     * determine the charset from HTML message.
                     */
                    if (mimeType.equalsIgnoreCase("text/html") && charset == null) {
                        InputStream in = part.getBody().getInputStream();
                        try {
                            byte[] buf = new byte[256];
                            in.read(buf, 0, buf.length);
                            String str = new String(buf, "US-ASCII");

                            if (str.isEmpty()) {
                                return "";
                            }
                            Pattern p = Pattern.compile("<meta http-equiv=\"?Content-Type\"? content=\"text/html; charset=(.+?)\">", Pattern.CASE_INSENSITIVE);
                            Matcher m = p.matcher(str);
                            if (m.find()) {
                                charset = m.group(1);
                            }
                        } finally {
                            try {
                                if (in instanceof BinaryTempFileBody.BinaryTempFileBodyInputStream) {
                                    /*
                                     * If this is a BinaryTempFileBodyInputStream, calling close()
                                     * will delete the file. But we can't let that happen because
                                     * the file needs to be opened again by the code a few lines
                                     * down.
                                     */
                                    ((BinaryTempFileBody.BinaryTempFileBodyInputStream) in).closeWithoutDeleting();
                                } else {
                                    in.close();
                                }
                            } catch (Exception e) { /* ignore */ }
                        }
                    }
                    charset = fixupCharset(charset, getMessageFromPart(part));

                    /*
                     * Now we read the part into a buffer for further processing. Because
                     * the stream is now wrapped we'll remove any transfer encoding at this point.
                     */
                    InputStream in = part.getBody().getInputStream();
                    try {
                        String text = CharsetSupport.readToString(in, charset);

                        // Replace the body with a TextBody that already contains the decoded text
                        part.setBody(new TextBody(text));<commit_msg>Properly decode the body in MessageExtractor.getTextFromPart()<commit_after>package com.fsck.k9.mail.internet;

import android.util.Log;

import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.BodyPart;
import com.fsck.k9.mail.Message;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.Multipart;
import com.fsck.k9.mail.Part;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.fsck.k9.mail.K9MailLib.LOG_TAG;
import static com.fsck.k9.mail.internet.CharsetSupport.fixupCharset;
import static com.fsck.k9.mail.internet.MimeUtility.getHeaderParameter;
import static com.fsck.k9.mail.internet.Viewable.Alternative;
import static com.fsck.k9.mail.internet.Viewable.Html;
import static com.fsck.k9.mail.internet.Viewable.MessageHeader;
import static com.fsck.k9.mail.internet.Viewable.Text;
import static com.fsck.k9.mail.internet.Viewable.Textual;

public class MessageExtractor {
    private MessageExtractor() {}

    public static String getTextFromPart(Part part) {
        try {
            if ((part != null) && (part.getBody() != null)) {
                final Body body = part.getBody();
                if (body instanceof TextBody) {
                    return ((TextBody)body).getText();
                }

                final String mimeType = part.getMimeType();
                if ((mimeType != null) && MimeUtility.mimeTypeMatches(mimeType, "text/*")) {
                    /*
                     * We've got a text part, so let's see if it needs to be processed further.
                     */
                    String charset = getHeaderParameter(part.getContentType(), "charset");
                    /*
                     * determine the charset from HTML message.
                     */
                    if (mimeType.equalsIgnoreCase("text/html") && charset == null) {
                        InputStream in = part.getBody().getInputStream();
                        try {
                            byte[] buf = new byte[256];
                            in.read(buf, 0, buf.length);
                            String str = new String(buf, "US-ASCII");

                            if (str.isEmpty()) {
                                return "";
                            }
                            Pattern p = Pattern.compile("<meta http-equiv=\"?Content-Type\"? content=\"text/html; charset=(.+?)\">", Pattern.CASE_INSENSITIVE);
                            Matcher m = p.matcher(str);
                            if (m.find()) {
                                charset = m.group(1);
                            }
                        } finally {
                            try {
                                if (in instanceof BinaryTempFileBody.BinaryTempFileBodyInputStream) {
                                    /*
                                     * If this is a BinaryTempFileBodyInputStream, calling close()
                                     * will delete the file. But we can't let that happen because
                                     * the file needs to be opened again by the code a few lines
                                     * down.
                                     */
                                    ((BinaryTempFileBody.BinaryTempFileBodyInputStream) in).closeWithoutDeleting();
                                } else {
                                    in.close();
                                }
                            } catch (Exception e) { /* ignore */ }
                        }
                    }
                    charset = fixupCharset(charset, getMessageFromPart(part));

                    /*
                     * Now we read the part into a buffer for further processing. Because
                     * the stream is now wrapped we'll remove any transfer encoding at this point.
                     */
                    InputStream in = MimeUtility.decodeBody(body);
                    try {
                        String text = CharsetSupport.readToString(in, charset);

                        // Replace the body with a TextBody that already contains the decoded text
                        part.setBody(new TextBody(text));
{'fraction_non_alphanumeric': 0.0927382053654024, 'fraction_numerical': 0.004856614246068455, 'mean_word_length': 1.325, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3405514', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2058, 'n_words': 502}