<commit_before># encoding: UTF-8

require 'rubygems'
require 'uuidtools'
require 'fileutils'
require 'mail'
require 'iconv'


module Attachments
  module Error
    class Base < StandardError; end
    class SaveFailed < Base; end
  end


  # Extract attachments of specific types from emails and save them in files
  class Extract
    UNCERTAIN_TYPES = [ "application/octet-stream" ].to_set

    def initialize include_types = [ "text/plain" ]
      @include_types = include_types
      reset
    end

    def close
      files.each do |f|
        FileUtils::rm(f[:tmpfile])
      end
      reset
    end

    def parse filename
      parse_file filename
    end

    def parse_file filename
      @last_parsed = filename

      # Load the email as binary to avoid encoding exceptions
      file = File.new(filename, "rb")
      raw_mail_data = file.read()
      file.close()

      parse_data raw_mail_data
    end

    def parse_data raw_mail_data
      @mail = Mail.new(raw_mail_data)

      # Parse parts recursively until it is not multipart
      # Ignore types that are not suited for forwarding
      parse_part @mail
    end

    def to 
      (@mail && @mail.to) || nil
    end

    def from
      (@mail && @mail.from) || nil
    end

    def subject
      (@mail && @mail.subject) || nil
    end

    def text_body
      if(@mail && @mail.text_part && @mail.text_part.body)
        m = @mail.text_part.body.decoded
        charset = @mail.text_part.charset
        charset ? Iconv.conv("utf-8", charset, m) : m
      else
        nil
      end
    end

    def html_body
      if(@mail && @mail.text_part && @mail.text_part.body)
        m = @mail.text_part.body.decoded<commit_msg>Force text_body encoding to utf-8<commit_after># encoding: UTF-8

require 'rubygems'
require 'uuidtools'
require 'fileutils'
require 'mail'
require 'iconv'


module Attachments
  module Error
    class Base < StandardError; end
    class SaveFailed < Base; end
  end


  # Extract attachments of specific types from emails and save them in files
  class Extract
    UNCERTAIN_TYPES = [ "application/octet-stream" ].to_set

    def initialize include_types = [ "text/plain" ]
      @include_types = include_types
      reset
    end

    def close
      files.each do |f|
        FileUtils::rm(f[:tmpfile])
      end
      reset
    end

    def parse filename
      parse_file filename
    end

    def parse_file filename
      @last_parsed = filename

      # Load the email as binary to avoid encoding exceptions
      file = File.new(filename, "rb")
      raw_mail_data = file.read()
      file.close()

      parse_data raw_mail_data
    end

    def parse_data raw_mail_data
      @mail = Mail.new(raw_mail_data)

      # Parse parts recursively until it is not multipart
      # Ignore types that are not suited for forwarding
      parse_part @mail
    end

    def to 
      (@mail && @mail.to) || nil
    end

    def from
      (@mail && @mail.from) || nil
    end

    def subject
      (@mail && @mail.subject) || nil
    end

    def text_body
      if(@mail && @mail.text_part && @mail.text_part.body)
        m = @mail.text_part.body.decoded
        charset = @mail.text_part.charset
        text = charset ? Iconv.conv("utf-8", charset, m) : m
        (text.respond_to? :force_encoding) ? text.force_encoding("utf-8") : text
      else
        nil
      end
    end

    def html_body
      if(@mail && @mail.text_part && @mail.text_part.body)
        m = @mail.text_part.body.decoded
{'fraction_non_alphanumeric': 0.08843732112192329, 'fraction_numerical': 0.0017172295363480253, 'mean_word_length': 2.306527909176916, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '743174', 'n_tokens_mistral': 1122, 'n_tokens_neox': 1071, 'n_words': 320}